001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_IOENGINE_KEY;
021import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
022import static org.apache.hadoop.hbase.io.ByteBuffAllocator.HEAP;
023import static org.junit.Assert.assertEquals;
024
025import java.nio.ByteBuffer;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.CombinedCacheStats;
031import org.apache.hadoop.hbase.nio.ByteBuff;
032import org.apache.hadoop.hbase.testclassification.SmallTests;
033import org.junit.Assert;
034import org.junit.ClassRule;
035import org.junit.Test;
036import org.junit.experimental.categories.Category;
037
038@Category({ SmallTests.class })
039public class TestCombinedBlockCache {
040
041  @ClassRule
042  public static final HBaseClassTestRule CLASS_RULE =
043    HBaseClassTestRule.forClass(TestCombinedBlockCache.class);
044
045  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
046
047  @Test
048  public void testCombinedCacheStats() {
049    CacheStats lruCacheStats = new CacheStats("lruCacheStats", 2);
050    CacheStats bucketCacheStats = new CacheStats("bucketCacheStats", 2);
051    CombinedCacheStats stats = new CombinedCacheStats(lruCacheStats, bucketCacheStats);
052
053    double delta = 0.01;
054
055    // period 1:
056    // lru cache: 1 hit caching, 1 miss caching
057    // bucket cache: 2 hit non-caching,1 miss non-caching/primary,1 fail insert
058    lruCacheStats.hit(true, true, BlockType.DATA);
059    lruCacheStats.miss(true, false, BlockType.DATA);
060    bucketCacheStats.hit(false, true, BlockType.DATA);
061    bucketCacheStats.hit(false, true, BlockType.DATA);
062    bucketCacheStats.miss(false, true, BlockType.DATA);
063
064    assertEquals(5, stats.getRequestCount());
065    assertEquals(2, stats.getRequestCachingCount());
066    assertEquals(2, stats.getMissCount());
067    assertEquals(1, stats.getPrimaryMissCount());
068    assertEquals(1, stats.getMissCachingCount());
069    assertEquals(3, stats.getHitCount());
070    assertEquals(3, stats.getPrimaryHitCount());
071    assertEquals(1, stats.getHitCachingCount());
072    assertEquals(0.6, stats.getHitRatio(), delta);
073    assertEquals(0.5, stats.getHitCachingRatio(), delta);
074    assertEquals(0.4, stats.getMissRatio(), delta);
075    assertEquals(0.5, stats.getMissCachingRatio(), delta);
076
077    // lru cache: 2 evicted, 1 evict
078    // bucket cache: 1 evict
079    lruCacheStats.evicted(1000, true);
080    lruCacheStats.evicted(1000, false);
081    lruCacheStats.evict();
082    bucketCacheStats.evict();
083    assertEquals(2, stats.getEvictionCount());
084    assertEquals(2, stats.getEvictedCount());
085    assertEquals(1, stats.getPrimaryEvictedCount());
086    assertEquals(1.0, stats.evictedPerEviction(), delta);
087
088    // lru cache: 1 fail insert
089    lruCacheStats.failInsert();
090    assertEquals(1, stats.getFailedInserts());
091
092    // rollMetricsPeriod
093    stats.rollMetricsPeriod();
094    assertEquals(3, stats.getSumHitCountsPastNPeriods());
095    assertEquals(5, stats.getSumRequestCountsPastNPeriods());
096    assertEquals(1, stats.getSumHitCachingCountsPastNPeriods());
097    assertEquals(2, stats.getSumRequestCachingCountsPastNPeriods());
098    assertEquals(0.6, stats.getHitRatioPastNPeriods(), delta);
099    assertEquals(0.5, stats.getHitCachingRatioPastNPeriods(), delta);
100
101    // period 2:
102    // lru cache: 3 hit caching
103    lruCacheStats.hit(true, true, BlockType.DATA);
104    lruCacheStats.hit(true, true, BlockType.DATA);
105    lruCacheStats.hit(true, true, BlockType.DATA);
106    stats.rollMetricsPeriod();
107    assertEquals(6, stats.getSumHitCountsPastNPeriods());
108    assertEquals(8, stats.getSumRequestCountsPastNPeriods());
109    assertEquals(4, stats.getSumHitCachingCountsPastNPeriods());
110    assertEquals(5, stats.getSumRequestCachingCountsPastNPeriods());
111    assertEquals(0.75, stats.getHitRatioPastNPeriods(), delta);
112    assertEquals(0.8, stats.getHitCachingRatioPastNPeriods(), delta);
113  }
114
115  @Test
116  public void testMultiThreadGetAndEvictBlock() throws Exception {
117    BlockCache blockCache = createCombinedBlockCache();
118    TestLruBlockCache.testMultiThreadGetAndEvictBlockInternal(blockCache);
119  }
120
121  @Test
122  public void testCombinedBlockCacheStatsWithDataBlockType() throws Exception {
123    testCombinedBlockCacheStats(BlockType.DATA, 0, 1);
124  }
125
126  @Test
127  public void testCombinedBlockCacheStatsWithMetaBlockType() throws Exception {
128    testCombinedBlockCacheStats(BlockType.META, 1, 0);
129  }
130
131  @Test
132  public void testCombinedBlockCacheStatsWithNoBlockType() throws Exception {
133    testCombinedBlockCacheStats(null, 0, 1);
134  }
135
136  private CombinedBlockCache createCombinedBlockCache() {
137    Configuration conf = UTIL.getConfiguration();
138    conf.set(BUCKET_CACHE_IOENGINE_KEY, "offheap");
139    conf.setInt(BUCKET_CACHE_SIZE_KEY, 32);
140    BlockCache blockCache = BlockCacheFactory.createBlockCache(conf);
141    Assert.assertTrue(blockCache instanceof CombinedBlockCache);
142    return (CombinedBlockCache) blockCache;
143  }
144
145  public void testCombinedBlockCacheStats(BlockType type, int expectedL1Miss, int expectedL2Miss)
146    throws Exception {
147    CombinedBlockCache blockCache = createCombinedBlockCache();
148    BlockCacheKey key = new BlockCacheKey("key1", 0, false, type);
149    int size = 100;
150    int length = HConstants.HFILEBLOCK_HEADER_SIZE + size;
151    byte[] byteArr = new byte[length];
152    HFileContext meta = new HFileContextBuilder().build();
153    HFileBlock blk = new HFileBlock(type != null ? type : BlockType.DATA, size, size, -1,
154      ByteBuff.wrap(ByteBuffer.wrap(byteArr, 0, size)), HFileBlock.FILL_HEADER, -1, 52, -1, meta,
155      HEAP);
156    blockCache.cacheBlock(key, blk);
157    blockCache.getBlock(key, true, false, true);
158    assertEquals(0, blockCache.getStats().getMissCount());
159    blockCache.evictBlock(key);
160    blockCache.getBlock(key, true, false, true);
161    assertEquals(1, blockCache.getStats().getMissCount());
162    assertEquals(expectedL1Miss, blockCache.getFirstLevelCache().getStats().getMissCount());
163    assertEquals(expectedL2Miss, blockCache.getSecondLevelCache().getStats().getMissCount());
164  }
165
166}