Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.apache.hadoop.hbase.io.hfile.FixedFileTrailer.createComparator;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertThrows;
024import static org.junit.Assert.assertTrue;
025import static org.junit.Assert.fail;
026
027import java.io.ByteArrayInputStream;
028import java.io.ByteArrayOutputStream;
029import java.io.DataInputStream;
030import java.io.DataOutputStream;
031import java.io.IOException;
032import java.util.ArrayList;
033import java.util.Collection;
034import java.util.List;
035import org.apache.hadoop.fs.FSDataInputStream;
036import org.apache.hadoop.fs.FSDataOutputStream;
037import org.apache.hadoop.fs.FileSystem;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.hbase.CellComparator;
040import org.apache.hadoop.hbase.CellComparatorImpl;
041import org.apache.hadoop.hbase.HBaseClassTestRule;
042import org.apache.hadoop.hbase.HBaseTestingUtil;
043import org.apache.hadoop.hbase.InnerStoreCellComparator;
044import org.apache.hadoop.hbase.MetaCellComparator;
045import org.apache.hadoop.hbase.testclassification.IOTests;
046import org.apache.hadoop.hbase.testclassification.SmallTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.junit.Before;
049import org.junit.ClassRule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052import org.junit.runner.RunWith;
053import org.junit.runners.Parameterized;
054import org.junit.runners.Parameterized.Parameters;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057
058@RunWith(Parameterized.class)
059@Category({ IOTests.class, SmallTests.class })
060public class TestFixedFileTrailer {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064    HBaseClassTestRule.forClass(TestFixedFileTrailer.class);
065
066  private static final Logger LOG = LoggerFactory.getLogger(TestFixedFileTrailer.class);
067  private static final int MAX_COMPARATOR_NAME_LENGTH = 128;
068
069  /**
070   * The number of used fields by version. Indexed by version minus two. Min version that we support
071   * is V2
072   */
073  private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 14, 15 };
074
075  private HBaseTestingUtil util = new HBaseTestingUtil();
076  private FileSystem fs;
077  private ByteArrayOutputStream baos = new ByteArrayOutputStream();
078  private int version;
079
080  static {
081    assert NUM_FIELDS_BY_VERSION.length == HFile.MAX_FORMAT_VERSION - HFile.MIN_FORMAT_VERSION + 1;
082  }
083
084  public TestFixedFileTrailer(int version) {
085    this.version = version;
086  }
087
088  @Parameters
089  public static Collection<Object[]> getParameters() {
090    List<Object[]> versionsToTest = new ArrayList<>();
091    for (int v = HFile.MIN_FORMAT_VERSION; v <= HFile.MAX_FORMAT_VERSION; ++v)
092      versionsToTest.add(new Integer[] { v });
093    return versionsToTest;
094  }
095
096  @Before
097  public void setUp() throws IOException {
098    fs = FileSystem.get(util.getConfiguration());
099  }
100
101  @Test
102  public void testCreateComparator() throws IOException {
103    assertEquals(InnerStoreCellComparator.class,
104      createComparator("org.apache.hadoop.hbase.KeyValue$KVComparator").getClass());
105    assertEquals(InnerStoreCellComparator.class,
106      createComparator(CellComparator.class.getName()).getClass());
107
108    assertEquals(MetaCellComparator.class,
109      createComparator("org.apache.hadoop.hbase.KeyValue$MetaComparator").getClass());
110    assertEquals(MetaCellComparator.class,
111      createComparator("org.apache.hadoop.hbase.CellComparator$MetaCellComparator").getClass());
112    assertEquals(MetaCellComparator.class,
113      createComparator("org.apache.hadoop.hbase.CellComparatorImpl$MetaCellComparator").getClass());
114    assertEquals(MetaCellComparator.class,
115      createComparator(MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass());
116    assertEquals(MetaCellComparator.META_COMPARATOR.getClass(),
117      createComparator(MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass());
118
119    assertEquals(CellComparatorImpl.COMPARATOR.getClass(),
120      createComparator(MetaCellComparator.COMPARATOR.getClass().getName()).getClass());
121
122    assertNull(createComparator(Bytes.BYTES_RAWCOMPARATOR.getClass().getName()));
123    assertNull(createComparator("org.apache.hadoop.hbase.KeyValue$RawBytesComparator"));
124
125    // Test an invalid comparatorClassName
126    assertThrows(IOException.class, () -> createComparator(""));
127  }
128
129  @Test
130  public void testTrailer() throws IOException {
131    FixedFileTrailer t = new FixedFileTrailer(version, HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION);
132    t.setDataIndexCount(3);
133    t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
134
135    t.setLastDataBlockOffset(291);
136    t.setNumDataIndexLevels(3);
137    t.setComparatorClass(InnerStoreCellComparator.INNER_STORE_COMPARATOR.getClass());
138    t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic.
139    t.setUncompressedDataIndexSize(827398717L); // Something random.
140
141    t.setLoadOnOpenOffset(128);
142    t.setMetaIndexCount(7);
143
144    t.setTotalUncompressedBytes(129731987);
145
146    {
147      DataOutputStream dos = new DataOutputStream(baos); // Limited scope.
148      t.serialize(dos);
149      dos.flush();
150      assertEquals(dos.size(), FixedFileTrailer.getTrailerSize(version));
151    }
152
153    byte[] bytes = baos.toByteArray();
154    baos.reset();
155
156    assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version));
157
158    ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
159
160    // Finished writing, trying to read.
161    {
162      DataInputStream dis = new DataInputStream(bais);
163      FixedFileTrailer t2 =
164        new FixedFileTrailer(version, HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION);
165      t2.deserialize(dis);
166      assertEquals(-1, bais.read()); // Ensure we have read everything.
167      checkLoadedTrailer(version, t, t2);
168    }
169
170    // Now check what happens if the trailer is corrupted.
171    Path trailerPath = new Path(util.getDataTestDir(), "trailer_" + version);
172
173    {
174      for (byte invalidVersion : new byte[] { HFile.MIN_FORMAT_VERSION - 1,
175        HFile.MAX_FORMAT_VERSION + 1 }) {
176        bytes[bytes.length - 1] = invalidVersion;
177        writeTrailer(trailerPath, null, bytes);
178        try {
179          readTrailer(trailerPath);
180          fail("Exception expected");
181        } catch (IllegalArgumentException ex) {
182          // Make it easy to debug this.
183          String msg = ex.getMessage();
184          String cleanMsg = msg.replaceAll("^(java(\\.[a-zA-Z]+)+:\\s+)?|\\s+\\(.*\\)\\s*$", "");
185          // will be followed by " expected: ..."
186          assertEquals("Actual exception message is \"" + msg + "\".\nCleaned-up message",
187            "Invalid HFile version: " + invalidVersion, cleanMsg);
188          LOG.info("Got an expected exception: " + msg);
189        }
190      }
191
192    }
193
194    // Now write the trailer into a file and auto-detect the version.
195    writeTrailer(trailerPath, t, null);
196
197    FixedFileTrailer t4 = readTrailer(trailerPath);
198
199    checkLoadedTrailer(version, t, t4);
200
201    String trailerStr = t.toString();
202    assertEquals(
203      "Invalid number of fields in the string representation " + "of the trailer: " + trailerStr,
204      NUM_FIELDS_BY_VERSION[version - 2], trailerStr.split(", ").length);
205    assertEquals(trailerStr, t4.toString());
206  }
207
208  @Test
209  public void testTrailerForV2NonPBCompatibility() throws Exception {
210    if (version == 2) {
211      FixedFileTrailer t = new FixedFileTrailer(version, HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM);
212      t.setDataIndexCount(3);
213      t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
214      t.setLastDataBlockOffset(291);
215      t.setNumDataIndexLevels(3);
216      t.setComparatorClass(CellComparatorImpl.COMPARATOR.getClass());
217      t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic.
218      t.setUncompressedDataIndexSize(827398717L); // Something random.
219      t.setLoadOnOpenOffset(128);
220      t.setMetaIndexCount(7);
221      t.setTotalUncompressedBytes(129731987);
222
223      {
224        DataOutputStream dos = new DataOutputStream(baos); // Limited scope.
225        serializeAsWritable(dos, t);
226        dos.flush();
227        assertEquals(FixedFileTrailer.getTrailerSize(version), dos.size());
228      }
229
230      byte[] bytes = baos.toByteArray();
231      baos.reset();
232      assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version));
233
234      ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
235      {
236        DataInputStream dis = new DataInputStream(bais);
237        FixedFileTrailer t2 =
238          new FixedFileTrailer(version, HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM);
239        t2.deserialize(dis);
240        assertEquals(-1, bais.read()); // Ensure we have read everything.
241        checkLoadedTrailer(version, t, t2);
242      }
243    }
244  }
245
246  // Copied from FixedFileTrailer for testing the reading part of
247  // FixedFileTrailer of non PB
248  // serialized FFTs.
249  private void serializeAsWritable(DataOutputStream output, FixedFileTrailer fft)
250    throws IOException {
251    BlockType.TRAILER.write(output);
252    output.writeLong(fft.getFileInfoOffset());
253    output.writeLong(fft.getLoadOnOpenDataOffset());
254    output.writeInt(fft.getDataIndexCount());
255    output.writeLong(fft.getUncompressedDataIndexSize());
256    output.writeInt(fft.getMetaIndexCount());
257    output.writeLong(fft.getTotalUncompressedBytes());
258    output.writeLong(fft.getEntryCount());
259    output.writeInt(fft.getCompressionCodec().ordinal());
260    output.writeInt(fft.getNumDataIndexLevels());
261    output.writeLong(fft.getFirstDataBlockOffset());
262    output.writeLong(fft.getLastDataBlockOffset());
263    Bytes.writeStringFixedSize(output, fft.getComparatorClassName(), MAX_COMPARATOR_NAME_LENGTH);
264    output
265      .writeInt(FixedFileTrailer.materializeVersion(fft.getMajorVersion(), fft.getMinorVersion()));
266  }
267
268  private FixedFileTrailer readTrailer(Path trailerPath) throws IOException {
269    FSDataInputStream fsdis = fs.open(trailerPath);
270    FixedFileTrailer trailerRead =
271      FixedFileTrailer.readFromStream(fsdis, fs.getFileStatus(trailerPath).getLen());
272    fsdis.close();
273    return trailerRead;
274  }
275
276  private void writeTrailer(Path trailerPath, FixedFileTrailer t, byte[] useBytesInstead)
277    throws IOException {
278    assert (t == null) != (useBytesInstead == null); // Expect one non-null.
279
280    FSDataOutputStream fsdos = fs.create(trailerPath);
281    fsdos.write(135); // to make deserializer's job less trivial
282    if (useBytesInstead != null) {
283      fsdos.write(useBytesInstead);
284    } else {
285      t.serialize(fsdos);
286    }
287    fsdos.close();
288  }
289
290  private void checkLoadedTrailer(int version, FixedFileTrailer expected, FixedFileTrailer loaded)
291    throws IOException {
292    assertEquals(version, loaded.getMajorVersion());
293    assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount());
294
295    assertEquals(
296      Math.min(expected.getEntryCount(), version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE),
297      loaded.getEntryCount());
298
299    if (version == 1) {
300      assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset());
301    }
302
303    if (version == 2) {
304      assertEquals(expected.getLastDataBlockOffset(), loaded.getLastDataBlockOffset());
305      assertEquals(expected.getNumDataIndexLevels(), loaded.getNumDataIndexLevels());
306      assertEquals(expected.createComparator().getClass().getName(),
307        loaded.createComparator().getClass().getName());
308      assertEquals(expected.getFirstDataBlockOffset(), loaded.getFirstDataBlockOffset());
309      assertTrue(expected.createComparator() instanceof CellComparatorImpl);
310      assertEquals(expected.getUncompressedDataIndexSize(), loaded.getUncompressedDataIndexSize());
311    }
312
313    assertEquals(expected.getLoadOnOpenDataOffset(), loaded.getLoadOnOpenDataOffset());
314    assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount());
315
316    assertEquals(expected.getTotalUncompressedBytes(), loaded.getTotalUncompressedBytes());
317  }
318
319}