001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.balancer;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.mockito.Mockito.mock;
023import static org.mockito.Mockito.when;
024
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Map;
030import java.util.Random;
031import java.util.Set;
032import java.util.TreeMap;
033import java.util.concurrent.ThreadLocalRandom;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.hbase.ClusterMetrics;
036import org.apache.hadoop.hbase.HBaseClassTestRule;
037import org.apache.hadoop.hbase.HBaseConfiguration;
038import org.apache.hadoop.hbase.HConstants;
039import org.apache.hadoop.hbase.RegionMetrics;
040import org.apache.hadoop.hbase.ServerMetrics;
041import org.apache.hadoop.hbase.ServerName;
042import org.apache.hadoop.hbase.Size;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.client.RegionInfo;
045import org.apache.hadoop.hbase.client.TableDescriptor;
046import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
047import org.apache.hadoop.hbase.master.MasterServices;
048import org.apache.hadoop.hbase.master.RegionPlan;
049import org.apache.hadoop.hbase.testclassification.LargeTests;
050import org.apache.hadoop.hbase.util.Bytes;
051import org.junit.BeforeClass;
052import org.junit.ClassRule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057
058import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
059
060@Category({ LargeTests.class })
061public class TestCacheAwareLoadBalancer extends BalancerTestBase {
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064    HBaseClassTestRule.forClass(TestCacheAwareLoadBalancer.class);
065
066  private static final Logger LOG = LoggerFactory.getLogger(TestCacheAwareLoadBalancer.class);
067
068  private static CacheAwareLoadBalancer loadBalancer;
069
070  static List<ServerName> servers;
071
072  static List<TableDescriptor> tableDescs;
073
074  static Map<TableName, String> tableMap = new HashMap<>();
075
076  static TableName[] tables = new TableName[] { TableName.valueOf("dt1"), TableName.valueOf("dt2"),
077    TableName.valueOf("dt3"), TableName.valueOf("dt4") };
078
079  private static List<ServerName> generateServers(int numServers) {
080    List<ServerName> servers = new ArrayList<>(numServers);
081    Random rand = ThreadLocalRandom.current();
082    for (int i = 0; i < numServers; i++) {
083      String host = "server" + rand.nextInt(100000);
084      int port = rand.nextInt(60000);
085      servers.add(ServerName.valueOf(host, port, -1));
086    }
087    return servers;
088  }
089
090  private static List<TableDescriptor> constructTableDesc(boolean hasBogusTable) {
091    List<TableDescriptor> tds = Lists.newArrayList();
092    for (int i = 0; i < tables.length; i++) {
093      TableDescriptor htd = TableDescriptorBuilder.newBuilder(tables[i]).build();
094      tds.add(htd);
095    }
096    return tds;
097  }
098
099  private ServerMetrics mockServerMetricsWithRegionCacheInfo(ServerName server,
100    List<RegionInfo> regionsOnServer, float currentCacheRatio, List<RegionInfo> oldRegionCacheInfo,
101    int oldRegionCachedSize, int regionSize) {
102    ServerMetrics serverMetrics = mock(ServerMetrics.class);
103    Map<byte[], RegionMetrics> regionLoadMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
104    for (RegionInfo info : regionsOnServer) {
105      RegionMetrics rl = mock(RegionMetrics.class);
106      when(rl.getReadRequestCount()).thenReturn(0L);
107      when(rl.getWriteRequestCount()).thenReturn(0L);
108      when(rl.getMemStoreSize()).thenReturn(Size.ZERO);
109      when(rl.getStoreFileSize()).thenReturn(Size.ZERO);
110      when(rl.getCurrentRegionCachedRatio()).thenReturn(currentCacheRatio);
111      when(rl.getRegionSizeMB()).thenReturn(new Size(regionSize, Size.Unit.MEGABYTE));
112      regionLoadMap.put(info.getRegionName(), rl);
113    }
114    when(serverMetrics.getRegionMetrics()).thenReturn(regionLoadMap);
115    Map<String, Integer> oldCacheRatioMap = new HashMap<>();
116    for (RegionInfo info : oldRegionCacheInfo) {
117      oldCacheRatioMap.put(info.getEncodedName(), oldRegionCachedSize);
118    }
119    when(serverMetrics.getRegionCachedInfo()).thenReturn(oldCacheRatioMap);
120    return serverMetrics;
121  }
122
123  @BeforeClass
124  public static void beforeAllTests() throws Exception {
125    servers = generateServers(3);
126    tableDescs = constructTableDesc(false);
127    Configuration conf = HBaseConfiguration.create();
128    conf.set(HConstants.BUCKET_CACHE_PERSISTENT_PATH_KEY, "prefetch_file_list");
129    loadBalancer = new CacheAwareLoadBalancer();
130    MasterServices services = mock(MasterServices.class);
131    when(services.getConfiguration()).thenReturn(conf);
132    loadBalancer.setMasterServices(services);
133    loadBalancer.loadConf(conf);
134  }
135
136  @Test
137  public void testRegionsNotCachedOnOldServerAndCurrentServer() throws Exception {
138    // The regions are not cached on old server as well as the current server. This causes
139    // skewness in the region allocation which should be fixed by the balancer
140
141    Map<ServerName, List<RegionInfo>> clusterState = new HashMap<>();
142    ServerName server0 = servers.get(0);
143    ServerName server1 = servers.get(1);
144    ServerName server2 = servers.get(2);
145
146    // Simulate that the regions previously hosted by server1 are now hosted on server0
147    List<RegionInfo> regionsOnServer0 = randomRegions(10);
148    List<RegionInfo> regionsOnServer1 = randomRegions(0);
149    List<RegionInfo> regionsOnServer2 = randomRegions(5);
150
151    clusterState.put(server0, regionsOnServer0);
152    clusterState.put(server1, regionsOnServer1);
153    clusterState.put(server2, regionsOnServer2);
154
155    // Mock cluster metrics
156    Map<ServerName, ServerMetrics> serverMetricsMap = new TreeMap<>();
157    serverMetricsMap.put(server0, mockServerMetricsWithRegionCacheInfo(server0, regionsOnServer0,
158      0.0f, new ArrayList<>(), 0, 10));
159    serverMetricsMap.put(server1, mockServerMetricsWithRegionCacheInfo(server1, regionsOnServer1,
160      0.0f, new ArrayList<>(), 0, 10));
161    serverMetricsMap.put(server2, mockServerMetricsWithRegionCacheInfo(server2, regionsOnServer2,
162      0.0f, new ArrayList<>(), 0, 10));
163    ClusterMetrics clusterMetrics = mock(ClusterMetrics.class);
164    when(clusterMetrics.getLiveServerMetrics()).thenReturn(serverMetricsMap);
165    loadBalancer.updateClusterMetrics(clusterMetrics);
166
167    Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable =
168      (Map) mockClusterServersWithTables(clusterState);
169    List<RegionPlan> plans = loadBalancer.balanceCluster(LoadOfAllTable);
170    Set<RegionInfo> regionsMovedFromServer0 = new HashSet<>();
171    Map<ServerName, List<RegionInfo>> targetServers = new HashMap<>();
172    for (RegionPlan plan : plans) {
173      if (plan.getSource().equals(server0)) {
174        regionsMovedFromServer0.add(plan.getRegionInfo());
175        if (!targetServers.containsKey(plan.getDestination())) {
176          targetServers.put(plan.getDestination(), new ArrayList<>());
177        }
178        targetServers.get(plan.getDestination()).add(plan.getRegionInfo());
179      }
180    }
181    // should move 5 regions from server0 to server 1
182    assertEquals(5, regionsMovedFromServer0.size());
183    assertEquals(5, targetServers.get(server1).size());
184  }
185
186  @Test
187  public void testRegionsPartiallyCachedOnOldServerAndNotCachedOnCurrentServer() throws Exception {
188    // The regions are partially cached on old server but not cached on the current server
189
190    Map<ServerName, List<RegionInfo>> clusterState = new HashMap<>();
191    ServerName server0 = servers.get(0);
192    ServerName server1 = servers.get(1);
193    ServerName server2 = servers.get(2);
194
195    // Simulate that the regions previously hosted by server1 are now hosted on server0
196    List<RegionInfo> regionsOnServer0 = randomRegions(10);
197    List<RegionInfo> regionsOnServer1 = randomRegions(0);
198    List<RegionInfo> regionsOnServer2 = randomRegions(5);
199
200    clusterState.put(server0, regionsOnServer0);
201    clusterState.put(server1, regionsOnServer1);
202    clusterState.put(server2, regionsOnServer2);
203
204    // Mock cluster metrics
205
206    // Mock 5 regions from server0 were previously hosted on server1
207    List<RegionInfo> oldCachedRegions = regionsOnServer0.subList(5, regionsOnServer0.size() - 1);
208
209    Map<ServerName, ServerMetrics> serverMetricsMap = new TreeMap<>();
210    serverMetricsMap.put(server0, mockServerMetricsWithRegionCacheInfo(server0, regionsOnServer0,
211      0.0f, new ArrayList<>(), 0, 10));
212    serverMetricsMap.put(server1, mockServerMetricsWithRegionCacheInfo(server1, regionsOnServer1,
213      0.0f, oldCachedRegions, 6, 10));
214    serverMetricsMap.put(server2, mockServerMetricsWithRegionCacheInfo(server2, regionsOnServer2,
215      0.0f, new ArrayList<>(), 0, 10));
216    ClusterMetrics clusterMetrics = mock(ClusterMetrics.class);
217    when(clusterMetrics.getLiveServerMetrics()).thenReturn(serverMetricsMap);
218    loadBalancer.updateClusterMetrics(clusterMetrics);
219
220    Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable =
221      (Map) mockClusterServersWithTables(clusterState);
222    List<RegionPlan> plans = loadBalancer.balanceCluster(LoadOfAllTable);
223    Set<RegionInfo> regionsMovedFromServer0 = new HashSet<>();
224    Map<ServerName, List<RegionInfo>> targetServers = new HashMap<>();
225    for (RegionPlan plan : plans) {
226      if (plan.getSource().equals(server0)) {
227        regionsMovedFromServer0.add(plan.getRegionInfo());
228        if (!targetServers.containsKey(plan.getDestination())) {
229          targetServers.put(plan.getDestination(), new ArrayList<>());
230        }
231        targetServers.get(plan.getDestination()).add(plan.getRegionInfo());
232      }
233    }
234    // should move 5 regions from server0 to server1
235    assertEquals(5, regionsMovedFromServer0.size());
236    assertEquals(5, targetServers.get(server1).size());
237    assertTrue(targetServers.get(server1).containsAll(oldCachedRegions));
238  }
239
240  @Test
241  public void testRegionsFullyCachedOnOldServerAndNotCachedOnCurrentServers() throws Exception {
242    // The regions are fully cached on old server
243
244    Map<ServerName, List<RegionInfo>> clusterState = new HashMap<>();
245    ServerName server0 = servers.get(0);
246    ServerName server1 = servers.get(1);
247    ServerName server2 = servers.get(2);
248
249    // Simulate that the regions previously hosted by server1 are now hosted on server0
250    List<RegionInfo> regionsOnServer0 = randomRegions(10);
251    List<RegionInfo> regionsOnServer1 = randomRegions(0);
252    List<RegionInfo> regionsOnServer2 = randomRegions(5);
253
254    clusterState.put(server0, regionsOnServer0);
255    clusterState.put(server1, regionsOnServer1);
256    clusterState.put(server2, regionsOnServer2);
257
258    // Mock cluster metrics
259
260    // Mock 5 regions from server0 were previously hosted on server1
261    List<RegionInfo> oldCachedRegions = regionsOnServer0.subList(5, regionsOnServer0.size() - 1);
262
263    Map<ServerName, ServerMetrics> serverMetricsMap = new TreeMap<>();
264    serverMetricsMap.put(server0, mockServerMetricsWithRegionCacheInfo(server0, regionsOnServer0,
265      0.0f, new ArrayList<>(), 0, 10));
266    serverMetricsMap.put(server1, mockServerMetricsWithRegionCacheInfo(server1, regionsOnServer1,
267      0.0f, oldCachedRegions, 10, 10));
268    serverMetricsMap.put(server2, mockServerMetricsWithRegionCacheInfo(server2, regionsOnServer2,
269      0.0f, new ArrayList<>(), 0, 10));
270    ClusterMetrics clusterMetrics = mock(ClusterMetrics.class);
271    when(clusterMetrics.getLiveServerMetrics()).thenReturn(serverMetricsMap);
272    loadBalancer.updateClusterMetrics(clusterMetrics);
273
274    Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable =
275      (Map) mockClusterServersWithTables(clusterState);
276    List<RegionPlan> plans = loadBalancer.balanceCluster(LoadOfAllTable);
277    Set<RegionInfo> regionsMovedFromServer0 = new HashSet<>();
278    Map<ServerName, List<RegionInfo>> targetServers = new HashMap<>();
279    for (RegionPlan plan : plans) {
280      if (plan.getSource().equals(server0)) {
281        regionsMovedFromServer0.add(plan.getRegionInfo());
282        if (!targetServers.containsKey(plan.getDestination())) {
283          targetServers.put(plan.getDestination(), new ArrayList<>());
284        }
285        targetServers.get(plan.getDestination()).add(plan.getRegionInfo());
286      }
287    }
288    // should move 5 regions from server0 to server1
289    assertEquals(5, regionsMovedFromServer0.size());
290    assertEquals(5, targetServers.get(server1).size());
291    assertTrue(targetServers.get(server1).containsAll(oldCachedRegions));
292  }
293
294  @Test
295  public void testRegionsFullyCachedOnOldAndCurrentServers() throws Exception {
296    // The regions are fully cached on old server
297
298    Map<ServerName, List<RegionInfo>> clusterState = new HashMap<>();
299    ServerName server0 = servers.get(0);
300    ServerName server1 = servers.get(1);
301    ServerName server2 = servers.get(2);
302
303    // Simulate that the regions previously hosted by server1 are now hosted on server0
304    List<RegionInfo> regionsOnServer0 = randomRegions(10);
305    List<RegionInfo> regionsOnServer1 = randomRegions(0);
306    List<RegionInfo> regionsOnServer2 = randomRegions(5);
307
308    clusterState.put(server0, regionsOnServer0);
309    clusterState.put(server1, regionsOnServer1);
310    clusterState.put(server2, regionsOnServer2);
311
312    // Mock cluster metrics
313
314    // Mock 5 regions from server0 were previously hosted on server1
315    List<RegionInfo> oldCachedRegions = regionsOnServer0.subList(5, regionsOnServer0.size() - 1);
316
317    Map<ServerName, ServerMetrics> serverMetricsMap = new TreeMap<>();
318    serverMetricsMap.put(server0, mockServerMetricsWithRegionCacheInfo(server0, regionsOnServer0,
319      1.0f, new ArrayList<>(), 0, 10));
320    serverMetricsMap.put(server1, mockServerMetricsWithRegionCacheInfo(server1, regionsOnServer1,
321      1.0f, oldCachedRegions, 10, 10));
322    serverMetricsMap.put(server2, mockServerMetricsWithRegionCacheInfo(server2, regionsOnServer2,
323      1.0f, new ArrayList<>(), 0, 10));
324    ClusterMetrics clusterMetrics = mock(ClusterMetrics.class);
325    when(clusterMetrics.getLiveServerMetrics()).thenReturn(serverMetricsMap);
326    loadBalancer.updateClusterMetrics(clusterMetrics);
327
328    Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable =
329      (Map) mockClusterServersWithTables(clusterState);
330    List<RegionPlan> plans = loadBalancer.balanceCluster(LoadOfAllTable);
331    Set<RegionInfo> regionsMovedFromServer0 = new HashSet<>();
332    Map<ServerName, List<RegionInfo>> targetServers = new HashMap<>();
333    for (RegionPlan plan : plans) {
334      if (plan.getSource().equals(server0)) {
335        regionsMovedFromServer0.add(plan.getRegionInfo());
336        if (!targetServers.containsKey(plan.getDestination())) {
337          targetServers.put(plan.getDestination(), new ArrayList<>());
338        }
339        targetServers.get(plan.getDestination()).add(plan.getRegionInfo());
340      }
341    }
342    // should move 5 regions from server0 to server1
343    assertEquals(5, regionsMovedFromServer0.size());
344    assertEquals(5, targetServers.get(server1).size());
345    assertTrue(targetServers.get(server1).containsAll(oldCachedRegions));
346  }
347
348  @Test
349  public void testRegionsPartiallyCachedOnOldServerAndCurrentServer() throws Exception {
350    // The regions are partially cached on old server
351
352    Map<ServerName, List<RegionInfo>> clusterState = new HashMap<>();
353    ServerName server0 = servers.get(0);
354    ServerName server1 = servers.get(1);
355    ServerName server2 = servers.get(2);
356
357    // Simulate that the regions previously hosted by server1 are now hosted on server0
358    List<RegionInfo> regionsOnServer0 = randomRegions(10);
359    List<RegionInfo> regionsOnServer1 = randomRegions(0);
360    List<RegionInfo> regionsOnServer2 = randomRegions(5);
361
362    clusterState.put(server0, regionsOnServer0);
363    clusterState.put(server1, regionsOnServer1);
364    clusterState.put(server2, regionsOnServer2);
365
366    // Mock cluster metrics
367
368    // Mock 5 regions from server0 were previously hosted on server1
369    List<RegionInfo> oldCachedRegions = regionsOnServer0.subList(5, regionsOnServer0.size() - 1);
370
371    Map<ServerName, ServerMetrics> serverMetricsMap = new TreeMap<>();
372    serverMetricsMap.put(server0, mockServerMetricsWithRegionCacheInfo(server0, regionsOnServer0,
373      0.2f, new ArrayList<>(), 0, 10));
374    serverMetricsMap.put(server1, mockServerMetricsWithRegionCacheInfo(server1, regionsOnServer1,
375      0.0f, oldCachedRegions, 6, 10));
376    serverMetricsMap.put(server2, mockServerMetricsWithRegionCacheInfo(server2, regionsOnServer2,
377      1.0f, new ArrayList<>(), 0, 10));
378    ClusterMetrics clusterMetrics = mock(ClusterMetrics.class);
379    when(clusterMetrics.getLiveServerMetrics()).thenReturn(serverMetricsMap);
380    loadBalancer.updateClusterMetrics(clusterMetrics);
381
382    Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable =
383      (Map) mockClusterServersWithTables(clusterState);
384    List<RegionPlan> plans = loadBalancer.balanceCluster(LoadOfAllTable);
385    Set<RegionInfo> regionsMovedFromServer0 = new HashSet<>();
386    Map<ServerName, List<RegionInfo>> targetServers = new HashMap<>();
387    for (RegionPlan plan : plans) {
388      if (plan.getSource().equals(server0)) {
389        regionsMovedFromServer0.add(plan.getRegionInfo());
390        if (!targetServers.containsKey(plan.getDestination())) {
391          targetServers.put(plan.getDestination(), new ArrayList<>());
392        }
393        targetServers.get(plan.getDestination()).add(plan.getRegionInfo());
394      }
395    }
396    assertEquals(5, regionsMovedFromServer0.size());
397    assertEquals(5, targetServers.get(server1).size());
398    assertTrue(targetServers.get(server1).containsAll(oldCachedRegions));
399  }
400}