001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.mockito.Mockito.mock;
022import static org.mockito.Mockito.when;
023
024import java.util.List;
025import java.util.OptionalDouble;
026import java.util.OptionalLong;
027import java.util.concurrent.atomic.AtomicInteger;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseConfiguration;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.HDFSBlocksDistribution;
033import org.apache.hadoop.hbase.ServerName;
034import org.apache.hadoop.hbase.client.RegionInfo;
035import org.apache.hadoop.hbase.ipc.RpcServerInterface;
036import org.apache.hadoop.hbase.testclassification.RegionServerTests;
037import org.apache.hadoop.hbase.testclassification.SmallTests;
038import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
039import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
040import org.apache.hadoop.hbase.wal.WALFactory;
041import org.junit.ClassRule;
042import org.junit.Test;
043import org.junit.experimental.categories.Category;
044import org.mockito.Mockito;
045import org.mockito.stubbing.Answer;
046
047import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
048
049@Category({ SmallTests.class, RegionServerTests.class })
050public class TestMetricsRegionServerAggregate {
051
052  @ClassRule
053  public static final HBaseClassTestRule CLASS_RULE =
054    HBaseClassTestRule.forClass(TestMetricsRegionServerAggregate.class);
055
056  @Test
057  public void test() {
058    AtomicInteger retVal = new AtomicInteger(0);
059    Answer defaultAnswer = invocation -> {
060      Class<?> returnType = invocation.getMethod().getReturnType();
061
062      if (returnType.equals(Integer.TYPE) || returnType.equals(Integer.class)) {
063        return retVal.get();
064      } else if (returnType.equals(Long.TYPE) || returnType.equals(Long.class)) {
065        return (long) retVal.get();
066      }
067      return Mockito.RETURNS_DEFAULTS.answer(invocation);
068    };
069
070    ServerName serverName = mock(ServerName.class);
071    when(serverName.getHostname()).thenReturn("foo");
072    WALFactory walFactory = mock(WALFactory.class);
073    RpcServerInterface rpcServer = mock(RpcServerInterface.class);
074    AtomicInteger storeFileCount = new AtomicInteger(1);
075    HRegion regionOne = getMockedRegion(defaultAnswer, "a", "foo", true, storeFileCount);
076    HRegion regionTwo = getMockedRegion(defaultAnswer, "b", "bar", true, storeFileCount);
077    HRegion regionThree = getMockedRegion(defaultAnswer, "c", "foo", false, storeFileCount);
078    HRegion regionFour = getMockedRegion(defaultAnswer, "d", "bar", false, storeFileCount);
079    List<HRegion> regions = Lists.newArrayList(regionOne, regionTwo, regionThree, regionFour);
080
081    int numStoresPerRegion = 2;
082    for (HRegion region : regions) {
083      // if adding more stores, update numStoresPerRegion so that tests below continue working
084      assertEquals(numStoresPerRegion, region.getStores().size());
085    }
086
087    HRegionServer regionServer = mock(HRegionServer.class, defaultAnswer);
088    when(regionServer.getWalFactory()).thenReturn(walFactory);
089    when(regionServer.getOnlineRegionsLocalContext()).thenReturn(regions);
090    when(regionServer.getServerName()).thenReturn(serverName);
091    Configuration conf = HBaseConfiguration.create();
092    int metricsPeriodSec = 600;
093    // set a very long period so that it doesn't actually run during our very quick test
094    conf.setLong(HConstants.REGIONSERVER_METRICS_PERIOD, metricsPeriodSec * 1000);
095    when(regionServer.getConfiguration()).thenReturn(conf);
096    when(regionServer.getRpcServer()).thenReturn(rpcServer);
097
098    MetricsRegionServerWrapperImpl wrapper = new MetricsRegionServerWrapperImpl(regionServer);
099
100    // we need to control the edge because rate calculations expect a
101    // stable interval relative to the configured period
102    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
103    EnvironmentEdgeManager.injectEdge(edge);
104
105    try {
106      for (int i = 1; i <= 10; i++) {
107        edge.incValue(wrapper.getPeriod());
108        retVal.incrementAndGet();
109        wrapper.forceRecompute();
110
111        int numRegions = regions.size();
112        int totalStores = numRegions * numStoresPerRegion;
113
114        // there are N regions, and each has M stores. everything gets aggregated, so
115        // multiply expected values accordingly
116        int expectedForRegions = retVal.get() * numRegions;
117        int expectedForStores = retVal.get() * totalStores;
118
119        assertEquals(totalStores, wrapper.getNumStores());
120        assertEquals(expectedForStores, wrapper.getFlushedCellsCount());
121        assertEquals(expectedForStores, wrapper.getCompactedCellsCount());
122        assertEquals(expectedForStores, wrapper.getMajorCompactedCellsCount());
123        assertEquals(expectedForStores, wrapper.getFlushedCellsSize());
124        assertEquals(expectedForStores, wrapper.getCompactedCellsSize());
125        assertEquals(expectedForStores, wrapper.getMajorCompactedCellsSize());
126        assertEquals(expectedForRegions, wrapper.getCellsCountCompactedFromMob());
127        assertEquals(expectedForRegions, wrapper.getCellsCountCompactedToMob());
128        assertEquals(expectedForRegions, wrapper.getCellsSizeCompactedFromMob());
129        assertEquals(expectedForRegions, wrapper.getCellsSizeCompactedToMob());
130        assertEquals(expectedForRegions, wrapper.getMobFlushCount());
131        assertEquals(expectedForRegions, wrapper.getMobFlushedCellsCount());
132        assertEquals(expectedForRegions, wrapper.getMobFlushedCellsSize());
133        assertEquals(expectedForRegions, wrapper.getMobScanCellsCount());
134        assertEquals(expectedForRegions, wrapper.getMobScanCellsSize());
135        assertEquals(expectedForRegions, wrapper.getCheckAndMutateChecksFailed());
136        assertEquals(expectedForRegions, wrapper.getCheckAndMutateChecksPassed());
137        assertEquals(expectedForStores, wrapper.getStoreFileIndexSize());
138        assertEquals(expectedForStores, wrapper.getTotalStaticIndexSize());
139        assertEquals(expectedForStores, wrapper.getTotalStaticBloomSize());
140        assertEquals(expectedForStores, wrapper.getBloomFilterRequestsCount());
141        assertEquals(expectedForStores, wrapper.getBloomFilterNegativeResultsCount());
142        assertEquals(expectedForStores, wrapper.getBloomFilterEligibleRequestsCount());
143        assertEquals(expectedForRegions, wrapper.getNumMutationsWithoutWAL());
144        assertEquals(expectedForRegions, wrapper.getDataInMemoryWithoutWAL());
145        assertEquals(expectedForRegions, wrapper.getAverageRegionSize());
146        assertEquals(expectedForRegions, wrapper.getBlockedRequestsCount());
147        assertEquals(expectedForStores, wrapper.getNumReferenceFiles());
148        assertEquals(expectedForStores, wrapper.getMemStoreSize());
149        assertEquals(expectedForStores, wrapper.getOnHeapMemStoreSize());
150        assertEquals(expectedForStores, wrapper.getOffHeapMemStoreSize());
151        assertEquals(expectedForStores, wrapper.getStoreFileSize());
152        assertEquals(expectedForRegions, wrapper.getReadRequestsCount());
153        assertEquals(expectedForRegions, wrapper.getCpRequestsCount());
154        assertEquals(expectedForRegions, wrapper.getFilteredReadRequestsCount());
155        assertEquals(expectedForRegions, wrapper.getWriteRequestsCount());
156        assertEquals(expectedForRegions * 2, wrapper.getTotalRowActionRequestCount());
157
158        // If we have N regions, each with M stores. That's N*M stores in total. In creating those
159        // stores, we increment the number and age of storefiles for each one. So the first
160        // store has 1 file of 1 age, then 2 files of 2 age, etc.
161        // formula for 1+2+3..+n
162        assertEquals((totalStores * (totalStores + 1)) / 2, wrapper.getNumStoreFiles());
163        assertEquals(totalStores, wrapper.getMaxStoreFiles());
164        assertEquals(totalStores, wrapper.getMaxStoreFileAge());
165        assertEquals(1, wrapper.getMinStoreFileAge());
166        assertEquals(totalStores / 2, wrapper.getAvgStoreFileAge());
167
168        // there are four regions, two are primary and the other two secondary
169        // for each type, one region has 100% locality, the other has 0%.
170        // this just proves we correctly aggregate for each
171        assertEquals(50.0, wrapper.getPercentFileLocal(), 0.0001);
172        assertEquals(50.0, wrapper.getPercentFileLocalSecondaryRegions(), 0.0001);
173
174        // readRequestCount and writeRequestCount are tracking the value of i, which increases by 1
175        // each interval. There are N regions, so the delta each interval is N*i=N. So the rate is
176        // simply N / period.
177        assertEquals((double) numRegions / metricsPeriodSec, wrapper.getReadRequestsRatePerSecond(),
178          0.0001);
179        assertEquals((double) numRegions / metricsPeriodSec,
180          wrapper.getWriteRequestsRatePerSecond(), 0.0001);
181        // total of above, so multiply by 2
182        assertEquals((double) numRegions / metricsPeriodSec * 2, wrapper.getRequestsPerSecond(),
183          0.0001);
184        // Similar logic to above, except there are M totalStores and each one is of
185        // size tracking i. So the rate is just M / period.
186        assertEquals((double) totalStores / metricsPeriodSec, wrapper.getStoreFileSizeGrowthRate(),
187          0.0001);
188      }
189    } finally {
190      EnvironmentEdgeManager.reset();
191    }
192  }
193
194  private HRegion getMockedRegion(Answer defaultAnswer, String name, String localOnHost,
195    boolean isPrimary, AtomicInteger storeFileCount) {
196    RegionInfo regionInfo = mock(RegionInfo.class);
197    when(regionInfo.getEncodedName()).thenReturn(name);
198    if (!isPrimary) {
199      when(regionInfo.getReplicaId()).thenReturn(RegionInfo.DEFAULT_REPLICA_ID + 1);
200    }
201    HDFSBlocksDistribution distribution = new HDFSBlocksDistribution();
202    distribution.addHostsAndBlockWeight(new String[] { localOnHost }, 100);
203
204    HStore store = getMockedStore(HStore.class, defaultAnswer, storeFileCount);
205    HMobStore mobStore = getMockedStore(HMobStore.class, defaultAnswer, storeFileCount);
206
207    HRegion region = mock(HRegion.class, defaultAnswer);
208    when(region.getRegionInfo()).thenReturn(regionInfo);
209    when(region.getHDFSBlocksDistribution()).thenReturn(distribution);
210    when(region.getStores()).thenReturn(Lists.newArrayList(store, mobStore));
211    return region;
212  }
213
214  private <T extends HStore> T getMockedStore(Class<T> clazz, Answer defaultAnswer,
215    AtomicInteger storeFileCount) {
216    T store = mock(clazz, defaultAnswer);
217    int storeFileCountVal = storeFileCount.getAndIncrement();
218    when(store.getStorefilesCount()).thenReturn(storeFileCountVal);
219    when(store.getAvgStoreFileAge()).thenReturn(OptionalDouble.of(storeFileCountVal));
220    when(store.getMaxStoreFileAge()).thenReturn(OptionalLong.of(storeFileCountVal));
221    when(store.getMinStoreFileAge()).thenReturn(OptionalLong.of(storeFileCountVal));
222    MemStoreSize memStore = mock(MemStoreSize.class, defaultAnswer);
223    when(store.getMemStoreSize()).thenReturn(memStore);
224    return store;
225  }
226
227}