001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotEquals;
023import static org.junit.Assert.assertNotNull;
024import static org.junit.Assert.assertNull;
025import static org.junit.Assert.assertTrue;
026import static org.junit.Assert.fail;
027
028import java.io.IOException;
029import java.util.Arrays;
030import java.util.Comparator;
031import java.util.Map;
032import org.apache.hadoop.fs.FSDataOutputStream;
033import org.apache.hadoop.fs.FileStatus;
034import org.apache.hadoop.fs.FileSystem;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.hbase.HBaseClassTestRule;
037import org.apache.hadoop.hbase.HBaseCommonTestingUtil;
038import org.apache.hadoop.hbase.HConstants;
039import org.apache.hadoop.hbase.TableDescriptors;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
042import org.apache.hadoop.hbase.client.TableDescriptor;
043import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
044import org.apache.hadoop.hbase.testclassification.MediumTests;
045import org.apache.hadoop.hbase.testclassification.MiscTests;
046import org.junit.AfterClass;
047import org.junit.Before;
048import org.junit.ClassRule;
049import org.junit.Rule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052import org.junit.rules.TestName;
053import org.slf4j.Logger;
054import org.slf4j.LoggerFactory;
055
056/**
057 * Tests for {@link FSTableDescriptors}.
058 */
059// Do not support to be executed in he same JVM as other tests
060@Category({ MiscTests.class, MediumTests.class })
061public class TestFSTableDescriptors {
062
063  @ClassRule
064  public static final HBaseClassTestRule CLASS_RULE =
065    HBaseClassTestRule.forClass(TestFSTableDescriptors.class);
066
067  private static final HBaseCommonTestingUtil UTIL = new HBaseCommonTestingUtil();
068  private static final Logger LOG = LoggerFactory.getLogger(TestFSTableDescriptors.class);
069
070  @Rule
071  public TestName name = new TestName();
072
073  private Path testDir;
074
075  @Before
076  public void setUp() {
077    testDir = UTIL.getDataTestDir(name.getMethodName());
078  }
079
080  @AfterClass
081  public static void tearDownAfterClass() {
082    UTIL.cleanupTestDir();
083  }
084
085  @Test(expected = IllegalArgumentException.class)
086  public void testRegexAgainstOldStyleTableInfo() {
087    Path p = new Path(testDir, FSTableDescriptors.TABLEINFO_FILE_PREFIX);
088    int i = FSTableDescriptors.getTableInfoSequenceIdAndFileLength(p).sequenceId;
089    assertEquals(0, i);
090    // Assert it won't eat garbage -- that it fails
091    p = new Path(testDir, "abc");
092    FSTableDescriptors.getTableInfoSequenceIdAndFileLength(p);
093  }
094
095  @Test
096  public void testCreateAndUpdate() throws IOException {
097    TableDescriptor htd =
098      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
099    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
100    FSTableDescriptors fstd = new FSTableDescriptors(fs, testDir);
101    assertTrue(fstd.createTableDescriptor(htd));
102    assertFalse(fstd.createTableDescriptor(htd));
103    Path tableInfoDir = new Path(CommonFSUtils.getTableDir(testDir, htd.getTableName()),
104      FSTableDescriptors.TABLEINFO_DIR);
105    FileStatus[] statuses = fs.listStatus(tableInfoDir);
106    assertEquals("statuses.length=" + statuses.length, 1, statuses.length);
107    for (int i = 0; i < 10; i++) {
108      fstd.update(htd);
109    }
110    statuses = fs.listStatus(tableInfoDir);
111    assertEquals(1, statuses.length);
112  }
113
114  @Test
115  public void testSequenceIdAdvancesOnTableInfo() throws IOException {
116    TableDescriptor htd =
117      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
118    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
119    FSTableDescriptors fstd = new FSTableDescriptors(fs, testDir);
120    Path previousPath = null;
121    int previousSeqId = -1;
122    for (int i = 0; i < 10; i++) {
123      Path path = fstd.updateTableDescriptor(htd);
124      int seqId = FSTableDescriptors.getTableInfoSequenceIdAndFileLength(path).sequenceId;
125      if (previousPath != null) {
126        // Assert we cleaned up the old file.
127        assertTrue(!fs.exists(previousPath));
128        assertEquals(previousSeqId + 1, seqId);
129      }
130      previousPath = path;
131      previousSeqId = seqId;
132    }
133  }
134
135  @Test
136  public void testFormatTableInfoSequenceId() {
137    Path p0 = assertWriteAndReadSequenceId(0);
138    // Assert p0 has format we expect.
139    StringBuilder sb = new StringBuilder();
140    for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) {
141      sb.append("0");
142    }
143    assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString() + ".0",
144      p0.getName());
145    // Check a few more.
146    Path p2 = assertWriteAndReadSequenceId(2);
147    Path p10000 = assertWriteAndReadSequenceId(10000);
148    // Get a .tablinfo that has no sequenceid suffix.
149    Path p = new Path(p0.getParent(), FSTableDescriptors.TABLEINFO_FILE_PREFIX);
150    FileStatus fs = new FileStatus(0, false, 0, 0, 0, p);
151    FileStatus fs0 = new FileStatus(0, false, 0, 0, 0, p0);
152    FileStatus fs2 = new FileStatus(0, false, 0, 0, 0, p2);
153    FileStatus fs10000 = new FileStatus(0, false, 0, 0, 0, p10000);
154    Comparator<FileStatus> comparator = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
155    assertTrue(comparator.compare(fs, fs0) > 0);
156    assertTrue(comparator.compare(fs0, fs2) > 0);
157    assertTrue(comparator.compare(fs2, fs10000) > 0);
158  }
159
160  private Path assertWriteAndReadSequenceId(final int i) {
161    Path p =
162      new Path(testDir, FSTableDescriptors.getTableInfoFileName(i, HConstants.EMPTY_BYTE_ARRAY));
163    int ii = FSTableDescriptors.getTableInfoSequenceIdAndFileLength(p).sequenceId;
164    assertEquals(i, ii);
165    return p;
166  }
167
168  @Test
169  public void testRemoves() throws IOException {
170    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
171    // Cleanup old tests if any detrius laying around.
172    TableDescriptors htds = new FSTableDescriptors(fs, testDir);
173    TableDescriptor htd =
174      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
175    htds.update(htd);
176    assertNotNull(htds.remove(htd.getTableName()));
177    assertNull(htds.remove(htd.getTableName()));
178  }
179
180  @Test
181  public void testReadingHTDFromFS() throws IOException {
182    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
183    TableDescriptor htd =
184      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
185    FSTableDescriptors fstd = new FSTableDescriptors(fs, testDir);
186    fstd.createTableDescriptor(htd);
187    TableDescriptor td2 =
188      FSTableDescriptors.getTableDescriptorFromFs(fs, testDir, htd.getTableName());
189    assertTrue(htd.equals(td2));
190  }
191
192  @Test
193  public void testTableDescriptors() throws IOException, InterruptedException {
194    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
195    // Cleanup old tests if any debris laying around.
196    FSTableDescriptors htds = new FSTableDescriptors(fs, testDir) {
197      @Override
198      public TableDescriptor get(TableName tablename) {
199        LOG.info(tablename + ", cachehits=" + this.cachehits);
200        return super.get(tablename);
201      }
202    };
203    final int count = 10;
204    // Write out table infos.
205    for (int i = 0; i < count; i++) {
206      htds.createTableDescriptor(
207        TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName() + i)).build());
208    }
209
210    for (int i = 0; i < count; i++) {
211      assertTrue(htds.get(TableName.valueOf(name.getMethodName() + i)) != null);
212    }
213    for (int i = 0; i < count; i++) {
214      assertTrue(htds.get(TableName.valueOf(name.getMethodName() + i)) != null);
215    }
216    // Update the table infos
217    for (int i = 0; i < count; i++) {
218      TableDescriptorBuilder builder =
219        TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName() + i));
220      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("" + i));
221      htds.update(builder.build());
222    }
223    // Wait a while so mod time we write is for sure different.
224    Thread.sleep(100);
225    for (int i = 0; i < count; i++) {
226      assertTrue(htds.get(TableName.valueOf(name.getMethodName() + i)) != null);
227    }
228    for (int i = 0; i < count; i++) {
229      assertTrue(htds.get(TableName.valueOf(name.getMethodName() + i)) != null);
230    }
231    assertEquals(count * 4, htds.invocations);
232    assertTrue("expected=" + (count * 2) + ", actual=" + htds.cachehits,
233      htds.cachehits >= (count * 2));
234  }
235
236  @Test
237  public void testTableDescriptorsNoCache() throws IOException, InterruptedException {
238    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
239    // Cleanup old tests if any debris laying around.
240    FSTableDescriptors htds = new FSTableDescriptorsTest(fs, testDir, false);
241    final int count = 10;
242    // Write out table infos.
243    for (int i = 0; i < count; i++) {
244      htds.createTableDescriptor(
245        TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName() + i)).build());
246    }
247
248    for (int i = 0; i < 2 * count; i++) {
249      assertNotNull("Expected HTD, got null instead",
250        htds.get(TableName.valueOf(name.getMethodName() + i % 2)));
251    }
252    // Update the table infos
253    for (int i = 0; i < count; i++) {
254      TableDescriptorBuilder builder =
255        TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName() + i));
256      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("" + i));
257      htds.update(builder.build());
258    }
259    for (int i = 0; i < count; i++) {
260      assertNotNull("Expected HTD, got null instead",
261        htds.get(TableName.valueOf(name.getMethodName() + i)));
262      assertTrue("Column Family " + i + " missing", htds
263        .get(TableName.valueOf(name.getMethodName() + i)).hasColumnFamily(Bytes.toBytes("" + i)));
264    }
265    assertEquals(count * 4, htds.invocations);
266    assertEquals("expected=0, actual=" + htds.cachehits, 0, htds.cachehits);
267  }
268
269  @Test
270  public void testGetAll() throws IOException, InterruptedException {
271    final String name = "testGetAll";
272    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
273    // Cleanup old tests if any debris laying around.
274    FSTableDescriptors htds = new FSTableDescriptorsTest(fs, testDir);
275    final int count = 4;
276    // Write out table infos.
277    for (int i = 0; i < count; i++) {
278      htds.createTableDescriptor(
279        TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build());
280    }
281    // add hbase:meta
282    htds
283      .createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.META_TABLE_NAME).build());
284    assertEquals("getAll() didn't return all TableDescriptors, expected: " + (count + 1) + " got: "
285      + htds.getAll().size(), count + 1, htds.getAll().size());
286  }
287
288  @Test
289  public void testParallelGetAll() throws IOException, InterruptedException {
290    final String name = "testParallelGetAll";
291    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
292    // Enable parallel load table descriptor.
293    FSTableDescriptors htds = new FSTableDescriptorsTest(fs, testDir, true, 20);
294    final int count = 100;
295    // Write out table infos.
296    for (int i = 0; i < count; i++) {
297      htds.createTableDescriptor(
298        TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build());
299    }
300    // add hbase:meta
301    htds
302      .createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.META_TABLE_NAME).build());
303
304    int getTableDescriptorSize = htds.getAll().size();
305    assertEquals("getAll() didn't return all TableDescriptors, expected: " + (count + 1) + " got: "
306      + getTableDescriptorSize, count + 1, getTableDescriptorSize);
307
308    // get again to check whether the cache works well
309    getTableDescriptorSize = htds.getAll().size();
310    assertEquals("getAll() didn't return all TableDescriptors with cache, expected: " + (count + 1)
311      + " got: " + getTableDescriptorSize, count + 1, getTableDescriptorSize);
312  }
313
314  @Test
315  public void testGetAllOrdering() throws Exception {
316    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
317    FSTableDescriptors tds = new FSTableDescriptorsTest(fs, testDir);
318
319    String[] tableNames = new String[] { "foo", "bar", "foo:bar", "bar:foo" };
320    for (String tableName : tableNames) {
321      tds.createTableDescriptor(
322        TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName)).build());
323    }
324
325    Map<String, TableDescriptor> tables = tds.getAll();
326    // Remove hbase:meta from list. It shows up now since we made it dynamic. The schema
327    // is written into the fs by the FSTableDescriptors constructor now where before it
328    // didn't.
329    tables.remove(TableName.META_TABLE_NAME.getNameAsString());
330    assertEquals(4, tables.size());
331
332    String[] tableNamesOrdered =
333      new String[] { "bar:foo", "default:bar", "default:foo", "foo:bar" };
334    int i = 0;
335    for (Map.Entry<String, TableDescriptor> entry : tables.entrySet()) {
336      assertEquals(tableNamesOrdered[i], entry.getKey());
337      assertEquals(tableNamesOrdered[i],
338        entry.getValue().getTableName().getNameWithNamespaceInclAsString());
339      i++;
340    }
341  }
342
343  @Test
344  public void testCacheConsistency() throws IOException, InterruptedException {
345    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
346    // Cleanup old tests if any debris laying around.
347    FSTableDescriptors chtds = new FSTableDescriptorsTest(fs, testDir);
348    FSTableDescriptors nonchtds = new FSTableDescriptorsTest(fs, testDir, false);
349
350    final int count = 10;
351    // Write out table infos via non-cached FSTableDescriptors
352    for (int i = 0; i < count; i++) {
353      nonchtds.createTableDescriptor(
354        TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName() + i)).build());
355    }
356
357    // Calls to getAll() won't increase the cache counter, do per table.
358    for (int i = 0; i < count; i++) {
359      assertTrue(chtds.get(TableName.valueOf(name.getMethodName() + i)) != null);
360    }
361
362    assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
363
364    // add a new entry for random table name.
365    TableName random = TableName.valueOf("random");
366    TableDescriptor htd = TableDescriptorBuilder.newBuilder(random).build();
367    nonchtds.createTableDescriptor(htd);
368
369    // random will only increase the cachehit by 1
370    assertEquals(nonchtds.getAll().size(), chtds.getAll().size() + 1);
371
372    for (Map.Entry<String, TableDescriptor> entry : chtds.getAll().entrySet()) {
373      String t = (String) entry.getKey();
374      TableDescriptor nchtd = entry.getValue();
375      assertTrue(
376        "expected " + htd.toString() + " got: " + chtds.get(TableName.valueOf(t)).toString(),
377        (nchtd.equals(chtds.get(TableName.valueOf(t)))));
378    }
379    // this is by design, for FSTableDescriptor with cache enabled, once we have done a full scan
380    // and load all the table descriptors to cache, we will not go to file system again, as the only
381    // way to update table descriptor is to through us so we can cache it when updating.
382    assertNotNull(nonchtds.get(random));
383    assertNull(chtds.get(random));
384  }
385
386  @Test
387  public void testNoSuchTable() throws IOException {
388    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
389    // Cleanup old tests if any detrius laying around.
390    TableDescriptors htds = new FSTableDescriptors(fs, testDir);
391    assertNull("There shouldn't be any HTD for this table",
392      htds.get(TableName.valueOf("NoSuchTable")));
393  }
394
395  @Test
396  public void testUpdates() throws IOException {
397    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
398    // Cleanup old tests if any detrius laying around.
399    TableDescriptors htds = new FSTableDescriptors(fs, testDir);
400    TableDescriptor htd =
401      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
402    htds.update(htd);
403    htds.update(htd);
404    htds.update(htd);
405  }
406
407  @Test
408  public void testTableInfoFileStatusComparator() {
409    FileStatus bare = new FileStatus(0, false, 0, 0, -1,
410      new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX));
411    FileStatus future = new FileStatus(0, false, 0, 0, -1,
412      new Path("/tmp/tablinfo." + EnvironmentEdgeManager.currentTime()));
413    FileStatus farFuture = new FileStatus(0, false, 0, 0, -1,
414      new Path("/tmp/tablinfo." + EnvironmentEdgeManager.currentTime() + 1000));
415    FileStatus[] alist = { bare, future, farFuture };
416    FileStatus[] blist = { bare, farFuture, future };
417    FileStatus[] clist = { farFuture, bare, future };
418    Comparator<FileStatus> c = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
419    Arrays.sort(alist, c);
420    Arrays.sort(blist, c);
421    Arrays.sort(clist, c);
422    // Now assert all sorted same in way we want.
423    for (int i = 0; i < alist.length; i++) {
424      assertTrue(alist[i].equals(blist[i]));
425      assertTrue(blist[i].equals(clist[i]));
426      assertTrue(clist[i].equals(i == 0 ? farFuture : i == 1 ? future : bare));
427    }
428  }
429
430  @Test
431  public void testReadingInvalidDirectoryFromFS() throws IOException {
432    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
433    try {
434      new FSTableDescriptors(fs, CommonFSUtils.getRootDir(UTIL.getConfiguration()))
435        .get(TableName.valueOf(HConstants.HBASE_TEMP_DIRECTORY));
436      fail("Shouldn't be able to read a table descriptor for the archive directory.");
437    } catch (Exception e) {
438      LOG.debug("Correctly got error when reading a table descriptor from the archive directory: "
439        + e.getMessage());
440    }
441  }
442
443  @Test
444  public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException {
445    TableDescriptor htd =
446      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
447    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
448    FSTableDescriptors fstd = new FSTableDescriptors(fs, testDir);
449    assertTrue(fstd.createTableDescriptor(htd));
450    assertFalse(fstd.createTableDescriptor(htd));
451    htd = TableDescriptorBuilder.newBuilder(htd)
452      .setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue")).build();
453    assertTrue(fstd.createTableDescriptor(htd)); // this will re-create
454    Path tableDir = CommonFSUtils.getTableDir(testDir, htd.getTableName());
455    assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir));
456  }
457
458  @Test
459  public void testIgnoreBrokenTableDescriptorFiles() throws IOException {
460    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
461      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")).build();
462    TableDescriptor newHtd =
463      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
464        .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf2")).build();
465    assertNotEquals(newHtd, htd);
466    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
467    FSTableDescriptors fstd = new FSTableDescriptors(fs, testDir, false, false);
468    fstd.update(htd);
469    byte[] bytes = TableDescriptorBuilder.toByteArray(newHtd);
470    Path tableDir = CommonFSUtils.getTableDir(testDir, htd.getTableName());
471    Path tableInfoDir = new Path(tableDir, FSTableDescriptors.TABLEINFO_DIR);
472    FileStatus[] statuses = fs.listStatus(tableInfoDir);
473    assertEquals(1, statuses.length);
474    int seqId =
475      FSTableDescriptors.getTableInfoSequenceIdAndFileLength(statuses[0].getPath()).sequenceId + 1;
476    Path brokenFile = new Path(tableInfoDir, FSTableDescriptors.getTableInfoFileName(seqId, bytes));
477    try (FSDataOutputStream out = fs.create(brokenFile)) {
478      out.write(bytes, 0, bytes.length / 2);
479    }
480    assertTrue(fs.exists(brokenFile));
481    TableDescriptor getTd = fstd.get(htd.getTableName());
482    assertEquals(htd, getTd);
483    assertFalse(fs.exists(brokenFile));
484  }
485
486  private static class FSTableDescriptorsTest extends FSTableDescriptors {
487
488    public FSTableDescriptorsTest(FileSystem fs, Path rootdir) {
489      this(fs, rootdir, true);
490    }
491
492    public FSTableDescriptorsTest(FileSystem fs, Path rootdir, boolean usecache) {
493      super(fs, rootdir, false, usecache);
494    }
495
496    public FSTableDescriptorsTest(FileSystem fs, Path rootdir, boolean usecache,
497      int tableDescriptorParallelLoadThreads) {
498      super(fs, rootdir, false, usecache, tableDescriptorParallelLoadThreads);
499    }
500
501    @Override
502    public TableDescriptor get(TableName tablename) {
503      LOG.info((super.isUsecache() ? "Cached" : "Non-Cached") + " TableDescriptor.get() on "
504        + tablename + ", cachehits=" + this.cachehits);
505      return super.get(tablename);
506    }
507  }
508}