001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static java.util.stream.Collectors.toList;
021import static org.apache.hadoop.hbase.HConstants.EMPTY_END_ROW;
022import static org.apache.hadoop.hbase.HConstants.EMPTY_START_ROW;
023import static org.apache.hadoop.hbase.client.RegionReplicaTestHelper.testLocator;
024import static org.hamcrest.CoreMatchers.instanceOf;
025import static org.hamcrest.MatcherAssert.assertThat;
026import static org.junit.Assert.assertArrayEquals;
027import static org.junit.Assert.assertEquals;
028import static org.junit.Assert.assertNotNull;
029import static org.junit.Assert.assertNull;
030import static org.junit.Assert.assertSame;
031
032import java.io.IOException;
033import java.util.Arrays;
034import java.util.Collection;
035import java.util.List;
036import java.util.concurrent.CompletableFuture;
037import java.util.concurrent.ExecutionException;
038import java.util.concurrent.ThreadLocalRandom;
039import java.util.stream.IntStream;
040import org.apache.hadoop.conf.Configuration;
041import org.apache.hadoop.hbase.CatalogReplicaMode;
042import org.apache.hadoop.hbase.HBaseClassTestRule;
043import org.apache.hadoop.hbase.HBaseTestingUtility;
044import org.apache.hadoop.hbase.HRegionLocation;
045import org.apache.hadoop.hbase.MetaTableAccessor;
046import org.apache.hadoop.hbase.NotServingRegionException;
047import org.apache.hadoop.hbase.RegionLocations;
048import org.apache.hadoop.hbase.ServerName;
049import org.apache.hadoop.hbase.TableName;
050import org.apache.hadoop.hbase.TableNotFoundException;
051import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
052import org.apache.hadoop.hbase.client.RegionReplicaTestHelper.Locator;
053import org.apache.hadoop.hbase.security.User;
054import org.apache.hadoop.hbase.testclassification.ClientTests;
055import org.apache.hadoop.hbase.testclassification.MediumTests;
056import org.apache.hadoop.hbase.util.Bytes;
057import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
058import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
059import org.junit.After;
060import org.junit.AfterClass;
061import org.junit.Before;
062import org.junit.BeforeClass;
063import org.junit.ClassRule;
064import org.junit.Test;
065import org.junit.experimental.categories.Category;
066import org.junit.runner.RunWith;
067import org.junit.runners.Parameterized;
068import org.junit.runners.Parameterized.Parameter;
069
070import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
071import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
072
073@Category({ MediumTests.class, ClientTests.class })
074@RunWith(Parameterized.class)
075public class TestAsyncNonMetaRegionLocator {
076
077  @ClassRule
078  public static final HBaseClassTestRule CLASS_RULE =
079    HBaseClassTestRule.forClass(TestAsyncNonMetaRegionLocator.class);
080
081  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
082
083  private static final TableName TABLE_NAME = TableName.valueOf("async");
084
085  private static byte[] FAMILY = Bytes.toBytes("cf");
086  private static final int NB_SERVERS = 4;
087  private static final int NUM_OF_META_REPLICA = NB_SERVERS - 1;
088
089  private static byte[][] SPLIT_KEYS;
090
091  private AsyncConnectionImpl conn;
092  private AsyncNonMetaRegionLocator locator;
093
094  @Parameter
095  public CatalogReplicaMode metaReplicaMode;
096
097  @BeforeClass
098  public static void setUp() throws Exception {
099    Configuration conf = TEST_UTIL.getConfiguration();
100
101    // Enable hbase:meta replication.
102    conf.setBoolean(ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_CATALOG_CONF_KEY, true);
103    conf.setLong("replication.source.sleepforretries", 10); // 10 ms
104
105    TEST_UTIL.startMiniCluster(NB_SERVERS);
106    Admin admin = TEST_UTIL.getAdmin();
107    admin.balancerSwitch(false, true);
108    // Enable hbase:meta replication.
109    HBaseTestingUtility.setReplicas(admin, TableName.META_TABLE_NAME, NUM_OF_META_REPLICA);
110    TEST_UTIL.waitFor(30000,
111      () -> TEST_UTIL.getMiniHBaseCluster().getRegions(TableName.META_TABLE_NAME).size()
112          >= NUM_OF_META_REPLICA);
113
114    SPLIT_KEYS = new byte[8][];
115    for (int i = 111; i < 999; i += 111) {
116      SPLIT_KEYS[i / 111 - 1] = Bytes.toBytes(String.format("%03d", i));
117    }
118  }
119
120  @AfterClass
121  public static void tearDown() throws Exception {
122    TEST_UTIL.shutdownMiniCluster();
123  }
124
125  @Before
126  public void setUpBeforeTest() throws InterruptedException, ExecutionException, IOException {
127    Configuration c = new Configuration(TEST_UTIL.getConfiguration());
128    // Enable meta replica LoadBalance mode for this connection.
129    c.set(RegionLocator.LOCATOR_META_REPLICAS_MODE, metaReplicaMode.toString());
130    ConnectionRegistry registry =
131      ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration(), User.getCurrent());
132    conn = new AsyncConnectionImpl(c, registry, registry.getClusterId().get(), User.getCurrent());
133    locator = new AsyncNonMetaRegionLocator(conn);
134  }
135
136  @After
137  public void tearDownAfterTest() throws IOException {
138    Admin admin = TEST_UTIL.getAdmin();
139    if (admin.tableExists(TABLE_NAME)) {
140      if (admin.isTableEnabled(TABLE_NAME)) {
141        admin.disableTable(TABLE_NAME);
142      }
143      admin.deleteTable(TABLE_NAME);
144    }
145    Closeables.close(conn, true);
146  }
147
148  @Parameterized.Parameters
149  public static Collection<Object[]> paramAbstractTestRegionLocatoreters() {
150    return Arrays
151      .asList(new Object[][] { { CatalogReplicaMode.NONE }, { CatalogReplicaMode.LOAD_BALANCE } });
152  }
153
154  private void createSingleRegionTable() throws IOException, InterruptedException {
155    TEST_UTIL.createTable(TABLE_NAME, FAMILY);
156    TEST_UTIL.waitTableAvailable(TABLE_NAME);
157  }
158
159  private CompletableFuture<HRegionLocation> getDefaultRegionLocation(TableName tableName,
160    byte[] row, RegionLocateType locateType, boolean reload) {
161    return locator
162      .getRegionLocations(tableName, row, RegionReplicaUtil.DEFAULT_REPLICA_ID, locateType, reload)
163      .thenApply(RegionLocations::getDefaultRegionLocation);
164  }
165
166  @Test
167  public void testNoTable() throws InterruptedException {
168    for (RegionLocateType locateType : RegionLocateType.values()) {
169      try {
170        getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, locateType, false).get();
171      } catch (ExecutionException e) {
172        assertThat(e.getCause(), instanceOf(TableNotFoundException.class));
173      }
174    }
175  }
176
177  @Test
178  public void testDisableTable() throws IOException, InterruptedException {
179    createSingleRegionTable();
180    TEST_UTIL.getAdmin().disableTable(TABLE_NAME);
181    for (RegionLocateType locateType : RegionLocateType.values()) {
182      try {
183        getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, locateType, false).get();
184      } catch (ExecutionException e) {
185        assertThat(e.getCause(), instanceOf(TableNotFoundException.class));
186      }
187    }
188  }
189
190  private void assertLocEquals(byte[] startKey, byte[] endKey, ServerName serverName,
191    HRegionLocation loc) {
192    RegionInfo info = loc.getRegion();
193    assertEquals(TABLE_NAME, info.getTable());
194    assertArrayEquals(startKey, info.getStartKey());
195    assertArrayEquals(endKey, info.getEndKey());
196    assertEquals(serverName, loc.getServerName());
197  }
198
199  @Test
200  public void testSingleRegionTable() throws IOException, InterruptedException, ExecutionException {
201    createSingleRegionTable();
202    ServerName serverName = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME).getServerName();
203    for (RegionLocateType locateType : RegionLocateType.values()) {
204      assertLocEquals(EMPTY_START_ROW, EMPTY_END_ROW, serverName,
205        getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, locateType, false).get());
206    }
207    byte[] key = new byte[ThreadLocalRandom.current().nextInt(128)];
208    Bytes.random(key);
209    for (RegionLocateType locateType : RegionLocateType.values()) {
210      assertLocEquals(EMPTY_START_ROW, EMPTY_END_ROW, serverName,
211        getDefaultRegionLocation(TABLE_NAME, key, locateType, false).get());
212    }
213  }
214
215  private void createMultiRegionTable() throws IOException, InterruptedException {
216    TEST_UTIL.createTable(TABLE_NAME, FAMILY, SPLIT_KEYS);
217    TEST_UTIL.waitTableAvailable(TABLE_NAME);
218  }
219
220  private static byte[][] getStartKeys() {
221    byte[][] startKeys = new byte[SPLIT_KEYS.length + 1][];
222    startKeys[0] = EMPTY_START_ROW;
223    System.arraycopy(SPLIT_KEYS, 0, startKeys, 1, SPLIT_KEYS.length);
224    return startKeys;
225  }
226
227  private static byte[][] getEndKeys() {
228    byte[][] endKeys = Arrays.copyOf(SPLIT_KEYS, SPLIT_KEYS.length + 1);
229    endKeys[endKeys.length - 1] = EMPTY_START_ROW;
230    return endKeys;
231  }
232
233  private ServerName[] getLocations(byte[][] startKeys) {
234    ServerName[] serverNames = new ServerName[startKeys.length];
235    TEST_UTIL.getHBaseCluster().getRegionServerThreads().stream().map(t -> t.getRegionServer())
236      .forEach(rs -> {
237        rs.getRegions(TABLE_NAME).forEach(r -> {
238          serverNames[Arrays.binarySearch(startKeys, r.getRegionInfo().getStartKey(),
239            Bytes::compareTo)] = rs.getServerName();
240        });
241      });
242    return serverNames;
243  }
244
245  @Test
246  public void testMultiRegionTable() throws IOException, InterruptedException {
247    createMultiRegionTable();
248    byte[][] startKeys = getStartKeys();
249    ServerName[] serverNames = getLocations(startKeys);
250    IntStream.range(0, 2).forEach(n -> IntStream.range(0, startKeys.length).forEach(i -> {
251      try {
252        assertLocEquals(startKeys[i], i == startKeys.length - 1 ? EMPTY_END_ROW : startKeys[i + 1],
253          serverNames[i],
254          getDefaultRegionLocation(TABLE_NAME, startKeys[i], RegionLocateType.CURRENT, false)
255            .get());
256      } catch (InterruptedException | ExecutionException e) {
257        throw new RuntimeException(e);
258      }
259    }));
260
261    locator.clearCache(TABLE_NAME);
262    IntStream.range(0, 2).forEach(n -> IntStream.range(0, startKeys.length).forEach(i -> {
263      try {
264        assertLocEquals(startKeys[i], i == startKeys.length - 1 ? EMPTY_END_ROW : startKeys[i + 1],
265          serverNames[i],
266          getDefaultRegionLocation(TABLE_NAME, startKeys[i], RegionLocateType.AFTER, false).get());
267      } catch (InterruptedException | ExecutionException e) {
268        throw new RuntimeException(e);
269      }
270    }));
271
272    locator.clearCache(TABLE_NAME);
273    byte[][] endKeys = getEndKeys();
274    IntStream.range(0, 2).forEach(
275      n -> IntStream.range(0, endKeys.length).map(i -> endKeys.length - 1 - i).forEach(i -> {
276        try {
277          assertLocEquals(i == 0 ? EMPTY_START_ROW : endKeys[i - 1], endKeys[i], serverNames[i],
278            getDefaultRegionLocation(TABLE_NAME, endKeys[i], RegionLocateType.BEFORE, false).get());
279        } catch (InterruptedException | ExecutionException e) {
280          throw new RuntimeException(e);
281        }
282      }));
283  }
284
285  @Test
286  public void testRegionMove() throws IOException, InterruptedException, ExecutionException {
287    createSingleRegionTable();
288    ServerName serverName = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME).getServerName();
289    HRegionLocation loc =
290      getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, RegionLocateType.CURRENT, false).get();
291    assertLocEquals(EMPTY_START_ROW, EMPTY_END_ROW, serverName, loc);
292    ServerName newServerName = TEST_UTIL.getHBaseCluster().getRegionServerThreads().stream()
293      .map(t -> t.getRegionServer().getServerName()).filter(sn -> !sn.equals(serverName)).findAny()
294      .get();
295
296    TEST_UTIL.getAdmin().move(Bytes.toBytes(loc.getRegion().getEncodedName()), newServerName);
297    while (
298      !TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME).getServerName().equals(newServerName)
299    ) {
300      Thread.sleep(100);
301    }
302    // Should be same as it is in cache
303    assertSame(loc,
304      getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, RegionLocateType.CURRENT, false).get());
305    locator.updateCachedLocationOnError(loc, null);
306    // null error will not trigger a cache cleanup
307    assertSame(loc,
308      getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, RegionLocateType.CURRENT, false).get());
309    locator.updateCachedLocationOnError(loc, new NotServingRegionException());
310    assertLocEquals(EMPTY_START_ROW, EMPTY_END_ROW, newServerName,
311      getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, RegionLocateType.CURRENT, false).get());
312  }
313
314  // usually locate after will return the same result, so we add a test to make it return different
315  // result.
316  @Test
317  public void testLocateAfter() throws IOException, InterruptedException, ExecutionException {
318    byte[] row = Bytes.toBytes("1");
319    byte[] splitKey = Arrays.copyOf(row, 2);
320    TEST_UTIL.createTable(TABLE_NAME, FAMILY, new byte[][] { splitKey });
321    TEST_UTIL.waitTableAvailable(TABLE_NAME);
322    HRegionLocation currentLoc =
323      getDefaultRegionLocation(TABLE_NAME, row, RegionLocateType.CURRENT, false).get();
324    ServerName currentServerName = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME).getServerName();
325    assertLocEquals(EMPTY_START_ROW, splitKey, currentServerName, currentLoc);
326
327    HRegionLocation afterLoc =
328      getDefaultRegionLocation(TABLE_NAME, row, RegionLocateType.AFTER, false).get();
329    ServerName afterServerName =
330      TEST_UTIL.getHBaseCluster().getRegionServerThreads().stream().map(t -> t.getRegionServer())
331        .filter(rs -> rs.getRegions(TABLE_NAME).stream()
332          .anyMatch(r -> Bytes.equals(splitKey, r.getRegionInfo().getStartKey())))
333        .findAny().get().getServerName();
334    assertLocEquals(splitKey, EMPTY_END_ROW, afterServerName, afterLoc);
335
336    assertSame(afterLoc,
337      getDefaultRegionLocation(TABLE_NAME, row, RegionLocateType.AFTER, false).get());
338  }
339
340  // For HBASE-17402
341  @Test
342  public void testConcurrentLocate() throws IOException, InterruptedException, ExecutionException {
343    createMultiRegionTable();
344    byte[][] startKeys = getStartKeys();
345    byte[][] endKeys = getEndKeys();
346    ServerName[] serverNames = getLocations(startKeys);
347    for (int i = 0; i < 100; i++) {
348      locator.clearCache(TABLE_NAME);
349      List<CompletableFuture<HRegionLocation>> futures =
350        IntStream.range(0, 1000).mapToObj(n -> String.format("%03d", n)).map(s -> Bytes.toBytes(s))
351          .map(r -> getDefaultRegionLocation(TABLE_NAME, r, RegionLocateType.CURRENT, false))
352          .collect(toList());
353      for (int j = 0; j < 1000; j++) {
354        int index = Math.min(8, j / 111);
355        assertLocEquals(startKeys[index], endKeys[index], serverNames[index], futures.get(j).get());
356      }
357    }
358  }
359
360  @Test
361  public void testReload() throws Exception {
362    createSingleRegionTable();
363    ServerName serverName = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME).getServerName();
364    for (RegionLocateType locateType : RegionLocateType.values()) {
365      assertLocEquals(EMPTY_START_ROW, EMPTY_END_ROW, serverName,
366        getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, locateType, false).get());
367    }
368    ServerName newServerName = TEST_UTIL.getHBaseCluster().getRegionServerThreads().stream()
369      .map(t -> t.getRegionServer().getServerName()).filter(sn -> !sn.equals(serverName)).findAny()
370      .get();
371    Admin admin = TEST_UTIL.getAdmin();
372    RegionInfo region = admin.getRegions(TABLE_NAME).stream().findAny().get();
373    admin.move(region.getEncodedNameAsBytes(), newServerName);
374    TEST_UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {
375
376      @Override
377      public boolean evaluate() throws Exception {
378        ServerName newServerName = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME).getServerName();
379        return newServerName != null && !newServerName.equals(serverName);
380      }
381
382      @Override
383      public String explainFailure() throws Exception {
384        return region.getRegionNameAsString() + " is still on " + serverName;
385      }
386
387    });
388    // The cached location will not change
389    for (RegionLocateType locateType : RegionLocateType.values()) {
390      assertLocEquals(EMPTY_START_ROW, EMPTY_END_ROW, serverName,
391        getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, locateType, false).get());
392    }
393    // should get the new location when reload = true
394    // when meta replica LoadBalance mode is enabled, it may delay a bit.
395    TEST_UTIL.waitFor(3000, new ExplainingPredicate<Exception>() {
396      @Override
397      public boolean evaluate() throws Exception {
398        HRegionLocation loc =
399          getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, RegionLocateType.CURRENT, true)
400            .get();
401        return newServerName.equals(loc.getServerName());
402      }
403
404      @Override
405      public String explainFailure() throws Exception {
406        return "New location does not show up in meta (replica) region";
407      }
408    });
409
410    // the cached location should be replaced
411    for (RegionLocateType locateType : RegionLocateType.values()) {
412      assertLocEquals(EMPTY_START_ROW, EMPTY_END_ROW, newServerName,
413        getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, locateType, false).get());
414    }
415  }
416
417  // Testcase for HBASE-20822
418  @Test
419  public void testLocateBeforeLastRegion()
420    throws IOException, InterruptedException, ExecutionException {
421    createMultiRegionTable();
422    getDefaultRegionLocation(TABLE_NAME, SPLIT_KEYS[0], RegionLocateType.CURRENT, false).join();
423    HRegionLocation loc =
424      getDefaultRegionLocation(TABLE_NAME, EMPTY_END_ROW, RegionLocateType.BEFORE, false).get();
425    // should locate to the last region
426    assertArrayEquals(loc.getRegion().getEndKey(), EMPTY_END_ROW);
427  }
428
429  @Test
430  public void testRegionReplicas() throws Exception {
431    TEST_UTIL.getAdmin().createTable(TableDescriptorBuilder.newBuilder(TABLE_NAME)
432      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).setRegionReplication(3).build());
433    TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME);
434    testLocator(TEST_UTIL, TABLE_NAME, new Locator() {
435
436      @Override
437      public void updateCachedLocationOnError(HRegionLocation loc, Throwable error)
438        throws Exception {
439        locator.updateCachedLocationOnError(loc, error);
440      }
441
442      @Override
443      public RegionLocations getRegionLocations(TableName tableName, int replicaId, boolean reload)
444        throws Exception {
445        return locator.getRegionLocations(tableName, EMPTY_START_ROW, replicaId,
446          RegionLocateType.CURRENT, reload).get();
447      }
448    });
449  }
450
451  // Testcase for HBASE-21961
452  @Test
453  public void testLocateBeforeInOnlyRegion() throws IOException, InterruptedException {
454    createSingleRegionTable();
455    HRegionLocation loc =
456      getDefaultRegionLocation(TABLE_NAME, Bytes.toBytes(1), RegionLocateType.BEFORE, false).join();
457    // should locate to the only region
458    assertArrayEquals(loc.getRegion().getStartKey(), EMPTY_START_ROW);
459    assertArrayEquals(loc.getRegion().getEndKey(), EMPTY_END_ROW);
460  }
461
462  @Test
463  public void testConcurrentUpdateCachedLocationOnError() throws Exception {
464    createSingleRegionTable();
465    HRegionLocation loc =
466      getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, RegionLocateType.CURRENT, false).get();
467    IntStream.range(0, 100).parallel()
468      .forEach(i -> locator.updateCachedLocationOnError(loc, new NotServingRegionException()));
469  }
470
471  @Test
472  public void testCacheLocationWhenGetAllLocations() throws Exception {
473    createMultiRegionTable();
474    AsyncConnectionImpl conn = (AsyncConnectionImpl) ConnectionFactory
475      .createAsyncConnection(TEST_UTIL.getConfiguration()).get();
476    conn.getRegionLocator(TABLE_NAME).getAllRegionLocations().get();
477    List<RegionInfo> regions = TEST_UTIL.getAdmin().getRegions(TABLE_NAME);
478    for (RegionInfo region : regions) {
479      assertNotNull(conn.getLocator().getNonMetaRegionLocator().getRegionLocationInCache(TABLE_NAME,
480        region.getStartKey()));
481    }
482  }
483
484  @Test
485  public void testDoNotCacheLocationWithNullServerNameWhenGetAllLocations() throws Exception {
486    createMultiRegionTable();
487    AsyncConnectionImpl conn = (AsyncConnectionImpl) ConnectionFactory
488      .createAsyncConnection(TEST_UTIL.getConfiguration()).get();
489    List<RegionInfo> regions = TEST_UTIL.getAdmin().getRegions(TABLE_NAME);
490    RegionInfo chosen = regions.get(0);
491
492    // re-populate region cache
493    AsyncTableRegionLocator regionLocator = conn.getRegionLocator(TABLE_NAME);
494    regionLocator.clearRegionLocationCache();
495    regionLocator.getAllRegionLocations().get();
496
497    // expect all to be non-null at first
498    int tries = 3;
499    checkRegionsWithRetries(conn, regions, null, tries);
500
501    // clear servername from region info
502    Put put = MetaTableAccessor.makePutFromRegionInfo(chosen, EnvironmentEdgeManager.currentTime());
503    MetaTableAccessor.addEmptyLocation(put, 0);
504    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Lists.newArrayList(put));
505
506    // re-populate region cache again. check that we succeeded in nulling the servername
507    regionLocator.clearRegionLocationCache();
508    for (HRegionLocation loc : regionLocator.getAllRegionLocations().get()) {
509      if (loc.getRegion().equals(chosen)) {
510        assertNull(loc.getServerName());
511      }
512    }
513
514    // expect all but chosen to be non-null. chosen should be null because serverName was null
515    checkRegionsWithRetries(conn, regions, chosen, tries);
516  }
517
518  // caching of getAllRegionLocations is async. so we give it a couple tries
519  private void checkRegionsWithRetries(AsyncConnectionImpl conn, List<RegionInfo> regions,
520    RegionInfo chosen, int retries) throws InterruptedException {
521    while (true) {
522      try {
523        checkRegions(conn, regions, chosen);
524        break;
525      } catch (AssertionError e) {
526        if (retries-- <= 0) {
527          throw e;
528        }
529        Thread.sleep(500);
530      }
531    }
532  }
533
534  private void checkRegions(AsyncConnectionImpl conn, List<RegionInfo> regions, RegionInfo chosen) {
535    for (RegionInfo region : regions) {
536      RegionLocations fromCache = conn.getLocator().getNonMetaRegionLocator()
537        .getRegionLocationInCache(TABLE_NAME, region.getStartKey());
538      if (region.equals(chosen)) {
539        assertNull(fromCache);
540      } else {
541        assertNotNull(fromCache);
542      }
543    }
544  }
545}