001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertTrue; 023import static org.junit.Assert.fail; 024 025import java.io.IOException; 026import java.lang.management.ManagementFactory; 027import java.util.Iterator; 028import java.util.List; 029import org.apache.hadoop.conf.Configuration; 030import org.apache.hadoop.fs.FileSystem; 031import org.apache.hadoop.hbase.ChoreService; 032import org.apache.hadoop.hbase.CoordinatedStateManager; 033import org.apache.hadoop.hbase.HBaseClassTestRule; 034import org.apache.hadoop.hbase.HBaseConfiguration; 035import org.apache.hadoop.hbase.HBaseTestingUtil; 036import org.apache.hadoop.hbase.HConstants; 037import org.apache.hadoop.hbase.Server; 038import org.apache.hadoop.hbase.ServerName; 039import org.apache.hadoop.hbase.Waiter; 040import org.apache.hadoop.hbase.client.AsyncClusterConnection; 041import org.apache.hadoop.hbase.client.Connection; 042import org.apache.hadoop.hbase.io.hfile.BlockCache; 043import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; 044import org.apache.hadoop.hbase.io.hfile.CacheStats; 045import org.apache.hadoop.hbase.io.hfile.Cacheable; 046import org.apache.hadoop.hbase.io.hfile.CachedBlock; 047import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache; 048import org.apache.hadoop.hbase.io.util.MemorySizeUtil; 049import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext; 050import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult; 051import org.apache.hadoop.hbase.testclassification.MediumTests; 052import org.apache.hadoop.hbase.testclassification.RegionServerTests; 053import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 054import org.junit.ClassRule; 055import org.junit.Test; 056import org.junit.experimental.categories.Category; 057 058@Category({ RegionServerTests.class, MediumTests.class }) 059public class TestHeapMemoryManager { 060 061 @ClassRule 062 public static final HBaseClassTestRule CLASS_RULE = 063 HBaseClassTestRule.forClass(TestHeapMemoryManager.class); 064 065 private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); 066 067 private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); 068 069 @Test 070 public void testAutoTunerShouldBeOffWhenMaxMinRangesForMemstoreIsNotGiven() throws Exception { 071 Configuration conf = HBaseConfiguration.create(); 072 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.02f); 073 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); 074 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.03f); 075 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 076 HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), 077 new MemstoreFlusherStub(0), new RegionServerStub(conf), regionServerAccounting); 078 assertFalse(manager.isTunerOn()); 079 } 080 081 @Test 082 public void testAutoTunerShouldBeOffWhenMaxMinRangesForBlockCacheIsNotGiven() throws Exception { 083 Configuration conf = HBaseConfiguration.create(); 084 conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.02f); 085 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 086 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.03f); 087 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 088 HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), 089 new MemstoreFlusherStub(0), new RegionServerStub(conf), regionServerAccounting); 090 assertFalse(manager.isTunerOn()); 091 } 092 093 @Test 094 public void testWhenMemstoreAndBlockCacheMaxMinChecksFails() throws Exception { 095 BlockCacheStub blockCache = new BlockCacheStub(0); 096 Configuration conf = HBaseConfiguration.create(); 097 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 098 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.06f); 099 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 100 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub(0); 101 try { 102 new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), 103 regionServerAccounting); 104 fail(); 105 } catch (RuntimeException e) { 106 } 107 conf = HBaseConfiguration.create(); 108 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.2f); 109 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 110 try { 111 new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), 112 regionServerAccounting); 113 fail(); 114 } catch (RuntimeException e) { 115 } 116 } 117 118 @Test 119 public void testWhenClusterIsWriteHeavyWithEmptyMemstore() throws Exception { 120 Configuration conf = HBaseConfiguration.create(); 121 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 122 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 123 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 124 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 125 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 126 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 127 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 128 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 129 // Empty block cache and memstore 130 blockCache.setTestBlockSize(0); 131 regionServerAccounting.setTestMemstoreSize(0); 132 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 133 // Let the system start with default values for memstore heap and block cache size. 134 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 135 new RegionServerStub(conf), regionServerAccounting); 136 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 137 long oldBlockCacheSize = blockCache.maxSize; 138 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 139 heapMemoryManager.start(choreService); 140 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 141 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 142 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 143 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 144 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 145 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 146 // Allow the tuner to run once and do necessary memory up 147 Thread.sleep(1500); 148 // No changes should be made by tuner as we already have lot of empty space 149 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 150 assertEquals(oldBlockCacheSize, blockCache.maxSize); 151 } 152 153 @Test 154 public void testHeapMemoryManagerWhenOffheapFlushesHappenUnderReadHeavyCase() throws Exception { 155 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 156 Configuration conf = HBaseConfiguration.create(); 157 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 158 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 159 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 160 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 161 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 162 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 163 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 164 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true); 165 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 166 // Empty memstore and but nearly filled block cache 167 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 168 regionServerAccounting.setTestMemstoreSize(0); 169 // Let the system start with default values for memstore heap and block cache size. 170 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 171 new RegionServerStub(conf), regionServerAccounting); 172 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 173 long oldBlockCacheSize = blockCache.maxSize; 174 float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE; 175 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 176 heapMemoryManager.start(choreService); 177 blockCache.evictBlock(null); 178 blockCache.evictBlock(null); 179 blockCache.evictBlock(null); 180 // do some offheap flushes also. So there should be decrease in memstore but 181 // not as that when we don't have offheap flushes 182 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 183 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 184 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 185 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 186 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 187 // Allow the tuner to run once and do necessary memory up 188 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 189 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 190 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 191 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 192 oldBlockCacheSize = blockCache.maxSize; 193 // Do some more evictions before the next run of HeapMemoryTuner 194 blockCache.evictBlock(null); 195 // Allow the tuner to run once and do necessary memory up 196 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 197 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 198 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 199 } 200 201 @Test 202 public void testHeapMemoryManagerWithOffheapMemstoreAndMixedWorkload() throws Exception { 203 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 204 Configuration conf = HBaseConfiguration.create(); 205 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 206 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 207 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 208 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 209 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 210 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 211 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 212 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true); 213 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 214 // Empty memstore and but nearly filled block cache 215 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 216 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 217 // Let the system start with default values for memstore heap and block cache size. 218 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 219 new RegionServerStub(conf), regionServerAccounting); 220 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 221 long oldBlockCacheSize = blockCache.maxSize; 222 float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE; 223 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 224 heapMemoryManager.start(choreService); 225 blockCache.evictBlock(null); 226 blockCache.evictBlock(null); 227 blockCache.evictBlock(null); 228 // do some offheap flushes also. So there should be decrease in memstore but 229 // not as that when we don't have offheap flushes 230 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 231 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 232 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 233 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 234 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 235 // Allow the tuner to run once and do necessary memory up 236 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 237 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 238 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 239 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 240 oldBlockCacheSize = blockCache.maxSize; 241 // change memstore size 242 // regionServerAccounting.setTestMemstoreSize((long)(maxHeapSize * 0.4 * 0.8)); 243 // The memstore size would have decreased. Now again do some flushes and ensure the 244 // flushes are due to onheap overhead. This should once again call for increase in 245 // memstore size but that increase should be to the safe size 246 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 247 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 248 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 249 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 250 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 251 // Allow the tuner to run once and do necessary memory up 252 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 253 assertHeapSpaceDelta(maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 254 assertHeapSpaceDelta(-maxStepValue, oldBlockCacheSize, blockCache.maxSize); 255 } 256 257 @Test 258 public void testWhenClusterIsReadHeavyWithEmptyBlockCache() throws Exception { 259 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 260 Configuration conf = HBaseConfiguration.create(); 261 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 262 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 263 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 264 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 265 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 266 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 267 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 268 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 269 // Empty block cache and memstore 270 blockCache.setTestBlockSize(0); 271 regionServerAccounting.setTestMemstoreSize(0); 272 // Let the system start with default values for memstore heap and block cache size. 273 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 274 new RegionServerStub(conf), regionServerAccounting); 275 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 276 long oldBlockCacheSize = blockCache.maxSize; 277 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 278 heapMemoryManager.start(choreService); 279 blockCache.evictBlock(null); 280 blockCache.evictBlock(null); 281 blockCache.evictBlock(null); 282 // Allow the tuner to run once and do necessary memory up 283 Thread.sleep(1500); 284 // No changes should be made by tuner as we already have lot of empty space 285 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 286 assertEquals(oldBlockCacheSize, blockCache.maxSize); 287 } 288 289 @Test 290 public void testWhenClusterIsWriteHeavy() throws Exception { 291 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 292 Configuration conf = HBaseConfiguration.create(); 293 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 294 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 295 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 296 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 297 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 298 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 299 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 300 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 301 // Empty block cache and but nearly filled memstore 302 blockCache.setTestBlockSize(0); 303 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 304 // Let the system start with default values for memstore heap and block cache size. 305 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 306 new RegionServerStub(conf), regionServerAccounting); 307 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 308 long oldBlockCacheSize = blockCache.maxSize; 309 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 310 heapMemoryManager.start(choreService); 311 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 312 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 313 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 314 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 315 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 316 // Allow the tuner to run once and do necessary memory up 317 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 318 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 319 memStoreFlusher.memstoreSize); 320 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 321 blockCache.maxSize); 322 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 323 oldBlockCacheSize = blockCache.maxSize; 324 // Do some more flushes before the next run of HeapMemoryTuner 325 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 326 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 327 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 328 // Allow the tuner to run once and do necessary memory up 329 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 330 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 331 memStoreFlusher.memstoreSize); 332 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 333 blockCache.maxSize); 334 } 335 336 @Test 337 public void testWhenClusterIsWriteHeavyWithOffheapMemstore() throws Exception { 338 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 339 Configuration conf = HBaseConfiguration.create(); 340 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 341 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 342 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 343 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 344 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 345 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 346 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 347 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 348 // Empty block cache and but nearly filled memstore 349 blockCache.setTestBlockSize(0); 350 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 351 // Let the system start with default values for memstore heap and block cache size. 352 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 353 new RegionServerStub(conf), regionServerAccounting); 354 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 355 long oldBlockCacheSize = blockCache.maxSize; 356 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 357 heapMemoryManager.start(choreService); 358 // this should not change anything with onheap memstore 359 memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK; 360 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 361 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 362 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 363 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 364 // Allow the tuner to run once and do necessary memory up 365 Thread.sleep(1500); 366 // No changes should be made by tuner as we already have lot of empty space 367 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 368 assertEquals(oldBlockCacheSize, blockCache.maxSize); 369 } 370 371 @Test 372 public void testWhenClusterIsReadHeavy() throws Exception { 373 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 374 Configuration conf = HBaseConfiguration.create(); 375 conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); 376 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 377 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 378 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 379 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 380 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 381 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 382 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 383 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 384 // Empty memstore and but nearly filled block cache 385 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 386 regionServerAccounting.setTestMemstoreSize(0); 387 // Let the system start with default values for memstore heap and block cache size. 388 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 389 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 390 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 391 long oldBlockCacheSize = blockCache.maxSize; 392 long oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; 393 long maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; 394 float maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; 395 maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 396 ? DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 397 : maxStepValue; 398 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 399 heapMemoryManager.start(choreService); 400 blockCache.evictBlock(null); 401 blockCache.evictBlock(null); 402 blockCache.evictBlock(null); 403 // Allow the tuner to run once and do necessary memory up 404 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 405 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 406 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 407 oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 408 oldBlockCacheSize = blockCache.maxSize; 409 oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; 410 maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; 411 maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; 412 maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 413 ? DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE 414 : maxStepValue; 415 // Do some more evictions before the next run of HeapMemoryTuner 416 blockCache.evictBlock(null); 417 // Allow the tuner to run once and do necessary memory up 418 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 419 assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 420 assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); 421 } 422 423 @Test 424 public void testWhenClusterIsHavingMoreWritesThanReads() throws Exception { 425 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 426 Configuration conf = HBaseConfiguration.create(); 427 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 428 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 429 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 430 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 431 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 432 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 433 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 434 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 435 // Both memstore and block cache are nearly filled 436 blockCache.setTestBlockSize(0); 437 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 438 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 439 // Let the system start with default values for memstore heap and block cache size. 440 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 441 new RegionServerStub(conf), regionServerAccounting); 442 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 443 long oldBlockCacheSize = blockCache.maxSize; 444 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 445 heapMemoryManager.start(choreService); 446 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 447 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 448 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 449 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 450 blockCache.evictBlock(null); 451 // Allow the tuner to run once and do necessary memory up 452 Thread.sleep(1500); 453 // No changes should happen as there is undefined increase in flushes and evictions 454 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 455 assertEquals(oldBlockCacheSize, blockCache.maxSize); 456 // Do some more flushes before the next run of HeapMemoryTuner 457 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 458 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 459 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 460 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 461 // Allow the tuner to run once and do necessary memory up 462 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 463 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 464 memStoreFlusher.memstoreSize); 465 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 466 blockCache.maxSize); 467 } 468 469 @Test 470 public void testBlockedFlushesIncreaseMemstoreInSteadyState() throws Exception { 471 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 472 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 473 Configuration conf = HBaseConfiguration.create(); 474 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); 475 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); 476 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 477 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); 478 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 479 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 480 RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); 481 // Both memstore and block cache are nearly filled 482 blockCache.setTestBlockSize(0); 483 regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8)); 484 blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); 485 // Let the system start with default values for memstore heap and block cache size. 486 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 487 new RegionServerStub(conf), regionServerAccounting); 488 long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; 489 long oldBlockCacheSize = blockCache.maxSize; 490 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 491 heapMemoryManager.start(choreService); 492 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK; 493 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 494 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 495 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 496 blockCache.evictBlock(null); 497 blockCache.evictBlock(null); 498 // Allow the tuner to run once and do necessary memory up 499 Thread.sleep(1500); 500 // No changes should happen as there is undefined increase in flushes and evictions 501 assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize); 502 assertEquals(oldBlockCacheSize, blockCache.maxSize); 503 // Flushes that block updates 504 memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK; 505 memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY); 506 blockCache.evictBlock(null); 507 blockCache.evictBlock(null); 508 blockCache.evictBlock(null); 509 blockCache.evictBlock(null); 510 // Allow the tuner to run once and do necessary memory up 511 Thread.sleep(1500); 512 assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, 513 memStoreFlusher.memstoreSize); 514 assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, 515 blockCache.maxSize); 516 } 517 518 @Test 519 public void testPluggingInHeapMemoryTuner() throws Exception { 520 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 521 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 522 Configuration conf = HBaseConfiguration.create(); 523 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.78f); 524 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f); 525 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); 526 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.02f); 527 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 528 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 529 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 530 HeapMemoryTuner.class); 531 // Let the system start with default values for memstore heap and block cache size. 532 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 533 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 534 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 535 heapMemoryManager.start(choreService); 536 // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner 537 CustomHeapMemoryTuner.memstoreSize = 0.78f; 538 CustomHeapMemoryTuner.blockCacheSize = 0.02f; 539 // Allow the tuner to run once and do necessary memory up 540 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 541 assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);// Memstore 542 assertHeapSpace(0.02f, blockCache.maxSize);// BlockCache 543 // Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner 544 CustomHeapMemoryTuner.blockCacheSize = 0.75f; 545 CustomHeapMemoryTuner.memstoreSize = 0.05f; 546 // Allow the tuner to run once and do necessary memory up 547 waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); 548 assertHeapSpace(0.75f, blockCache.maxSize);// BlockCache 549 assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);// Memstore 550 } 551 552 @Test 553 public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception { 554 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 555 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 556 Configuration conf = HBaseConfiguration.create(); 557 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f); 558 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f); 559 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 560 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f); 561 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 562 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 563 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 564 HeapMemoryTuner.class); 565 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 566 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 567 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 568 heapMemoryManager.start(choreService); 569 CustomHeapMemoryTuner.memstoreSize = 0.78f; 570 CustomHeapMemoryTuner.blockCacheSize = 0.02f; 571 Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up 572 // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the 573 // upper bound. Same with block cache as 10% is the lower bound. 574 assertHeapSpace(0.7f, memStoreFlusher.memstoreSize); 575 assertHeapSpace(0.1f, blockCache.maxSize); 576 } 577 578 @Test 579 public void testWhenCombinedHeapSizesFromTunerGoesOutSideMaxLimit() throws Exception { 580 BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); 581 MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); 582 Configuration conf = HBaseConfiguration.create(); 583 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f); 584 conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f); 585 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); 586 conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f); 587 conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); 588 conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); 589 conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, 590 HeapMemoryTuner.class); 591 HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, 592 new RegionServerStub(conf), new RegionServerAccountingStub(conf)); 593 long oldMemstoreSize = memStoreFlusher.memstoreSize; 594 long oldBlockCacheSize = blockCache.maxSize; 595 final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); 596 heapMemoryManager.start(choreService); 597 CustomHeapMemoryTuner.memstoreSize = 0.7f; 598 CustomHeapMemoryTuner.blockCacheSize = 0.3f; 599 // Allow the tuner to run once and do necessary memory up 600 Thread.sleep(1500); 601 assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize); 602 assertEquals(oldBlockCacheSize, blockCache.maxSize); 603 } 604 605 private void assertHeapSpace(float expectedHeapPercentage, long currentHeapSpace) { 606 long expected = (long) (this.maxHeapSize * expectedHeapPercentage); 607 assertEquals(expected, currentHeapSpace); 608 } 609 610 private void assertHeapSpaceDelta(double expectedDeltaPercent, long oldHeapSpace, 611 long newHeapSpace) { 612 double expctedMinDelta = (double) (this.maxHeapSize * expectedDeltaPercent); 613 // Tolerable error 614 double error = 0.95; 615 if (expectedDeltaPercent > 0) { 616 assertTrue(expctedMinDelta * error <= (double) (newHeapSpace - oldHeapSpace)); 617 assertTrue(expctedMinDelta / error >= (double) (newHeapSpace - oldHeapSpace)); 618 } else { 619 assertTrue(-expctedMinDelta * error <= (double) (oldHeapSpace - newHeapSpace)); 620 assertTrue(-expctedMinDelta / error >= (double) (oldHeapSpace - newHeapSpace)); 621 } 622 } 623 624 private void waitForTune(final MemstoreFlusherStub memStoreFlusher, 625 final long oldMemstoreHeapSize) throws Exception { 626 // Allow the tuner to run once and do necessary memory up 627 UTIL.waitFor(10000, new Waiter.Predicate<Exception>() { 628 @Override 629 public boolean evaluate() throws Exception { 630 return oldMemstoreHeapSize != memStoreFlusher.memstoreSize; 631 } 632 }); 633 } 634 635 private static class BlockCacheStub implements ResizableBlockCache { 636 CacheStats stats = new CacheStats("test"); 637 long maxSize = 0; 638 private long testBlockSize = 0; 639 640 public BlockCacheStub(long size) { 641 this.maxSize = size; 642 } 643 644 @Override 645 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { 646 647 } 648 649 @Override 650 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { 651 652 } 653 654 @Override 655 public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, 656 boolean updateCacheMetrics) { 657 return null; 658 } 659 660 @Override 661 public boolean evictBlock(BlockCacheKey cacheKey) { 662 stats.evicted(0, cacheKey != null ? cacheKey.isPrimary() : true); 663 return false; 664 } 665 666 @Override 667 public int evictBlocksByHfileName(String hfileName) { 668 stats.evicted(0, true); // Just assuming only one block for file here. 669 return 0; 670 } 671 672 @Override 673 public CacheStats getStats() { 674 return this.stats; 675 } 676 677 @Override 678 public void shutdown() { 679 680 } 681 682 @Override 683 public long size() { 684 return 0; 685 } 686 687 @Override 688 public long getMaxSize() { 689 return 0; 690 } 691 692 @Override 693 public long getFreeSize() { 694 return 0; 695 } 696 697 @Override 698 public long getCurrentSize() { 699 return this.testBlockSize; 700 } 701 702 @Override 703 public long getCurrentDataSize() { 704 return 0; 705 } 706 707 @Override 708 public long getBlockCount() { 709 return 0; 710 } 711 712 @Override 713 public long getDataBlockCount() { 714 return 0; 715 } 716 717 @Override 718 public void setMaxSize(long size) { 719 this.maxSize = size; 720 } 721 722 @Override 723 public Iterator<CachedBlock> iterator() { 724 return null; 725 } 726 727 @Override 728 public BlockCache[] getBlockCaches() { 729 return null; 730 } 731 732 public void setTestBlockSize(long testBlockSize) { 733 this.testBlockSize = testBlockSize; 734 } 735 } 736 737 private static class MemstoreFlusherStub implements FlushRequester { 738 739 long memstoreSize; 740 741 FlushRequestListener listener; 742 743 FlushType flushType = FlushType.NORMAL; 744 745 public MemstoreFlusherStub(long memstoreSize) { 746 this.memstoreSize = memstoreSize; 747 } 748 749 @Override 750 public boolean requestFlush(HRegion region, FlushLifeCycleTracker tracker) { 751 this.listener.flushRequested(flushType, region); 752 return true; 753 } 754 755 @Override 756 public boolean requestFlush(HRegion region, List<byte[]> families, 757 FlushLifeCycleTracker tracker) { 758 return true; 759 } 760 761 @Override 762 public boolean requestDelayedFlush(HRegion region, long delay) { 763 return true; 764 } 765 766 @Override 767 public void registerFlushRequestListener(FlushRequestListener listener) { 768 this.listener = listener; 769 } 770 771 @Override 772 public boolean unregisterFlushRequestListener(FlushRequestListener listener) { 773 return false; 774 } 775 776 @Override 777 public void setGlobalMemStoreLimit(long globalMemStoreSize) { 778 this.memstoreSize = globalMemStoreSize; 779 } 780 } 781 782 private static class RegionServerStub implements Server { 783 private Configuration conf; 784 private boolean stopped = false; 785 786 public RegionServerStub(Configuration conf) { 787 this.conf = conf; 788 } 789 790 @Override 791 public void abort(String why, Throwable e) { 792 793 } 794 795 @Override 796 public boolean isAborted() { 797 return false; 798 } 799 800 @Override 801 public void stop(String why) { 802 this.stopped = true; 803 } 804 805 @Override 806 public boolean isStopped() { 807 return this.stopped; 808 } 809 810 @Override 811 public Configuration getConfiguration() { 812 return this.conf; 813 } 814 815 @Override 816 public ZKWatcher getZooKeeper() { 817 return null; 818 } 819 820 @Override 821 public CoordinatedStateManager getCoordinatedStateManager() { 822 return null; 823 } 824 825 @Override 826 public Connection getConnection() { 827 return null; 828 } 829 830 @Override 831 public ServerName getServerName() { 832 return ServerName.valueOf("server1", 4000, 12345); 833 } 834 835 @Override 836 public ChoreService getChoreService() { 837 return null; 838 } 839 840 @Override 841 public FileSystem getFileSystem() { 842 return null; 843 } 844 845 @Override 846 public boolean isStopping() { 847 return false; 848 } 849 850 @Override 851 public Connection createConnection(Configuration conf) throws IOException { 852 return null; 853 } 854 855 @Override 856 public AsyncClusterConnection getAsyncClusterConnection() { 857 return null; 858 } 859 } 860 861 static class CustomHeapMemoryTuner implements HeapMemoryTuner { 862 static float blockCacheSize = 0.4f; 863 static float memstoreSize = 0.4f; 864 865 @Override 866 public Configuration getConf() { 867 return null; 868 } 869 870 @Override 871 public void setConf(Configuration arg0) { 872 873 } 874 875 @Override 876 public TunerResult tune(TunerContext context) { 877 TunerResult result = new TunerResult(true); 878 result.setBlockCacheSize(blockCacheSize); 879 result.setMemStoreSize(memstoreSize); 880 return result; 881 } 882 } 883 884 private static class RegionServerAccountingStub extends RegionServerAccounting { 885 boolean offheap; 886 887 public RegionServerAccountingStub(Configuration conf) { 888 super(conf); 889 } 890 891 public RegionServerAccountingStub(Configuration conf, boolean offheap) { 892 super(conf); 893 this.offheap = offheap; 894 } 895 896 private long testMemstoreSize = 0; 897 898 @Override 899 public long getGlobalMemStoreDataSize() { 900 return testMemstoreSize; 901 } 902 903 @Override 904 public long getGlobalMemStoreHeapSize() { 905 return testMemstoreSize; 906 } 907 908 @Override 909 public boolean isOffheap() { 910 return offheap; 911 } 912 913 public void setTestMemstoreSize(long testMemstoreSize) { 914 this.testMemstoreSize = testMemstoreSize; 915 } 916 } 917}