001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.wal;
019
020import static org.hamcrest.MatcherAssert.assertThat;
021import static org.hamcrest.Matchers.hasSize;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertTrue;
024
025import java.util.Arrays;
026import java.util.List;
027import java.util.NavigableMap;
028import java.util.TreeMap;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.Cell;
031import org.apache.hadoop.hbase.CellBuilderType;
032import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
033import org.apache.hadoop.hbase.HBaseTestingUtil;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.client.RegionInfoBuilder;
037import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
038import org.apache.hadoop.hbase.util.Bytes;
039import org.junit.Test;
040import org.slf4j.Logger;
041import org.slf4j.LoggerFactory;
042
043@SuppressWarnings("checkstyle:innerassignment")
044public abstract class CompressedWALTestBase {
045  private static final Logger LOG = LoggerFactory.getLogger(CompressedWALTestBase.class);
046
047  protected final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
048
049  static final byte[] VALUE;
050  static {
051    // 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987, 1597
052    VALUE =
053      new byte[1 + 1 + 2 + 3 + 5 + 8 + 13 + 21 + 34 + 55 + 89 + 144 + 233 + 377 + 610 + 987 + 1597];
054    int off = 0;
055    Arrays.fill(VALUE, off, (off += 1), (byte) 'A');
056    Arrays.fill(VALUE, off, (off += 1), (byte) 'B');
057    Arrays.fill(VALUE, off, (off += 2), (byte) 'C');
058    Arrays.fill(VALUE, off, (off += 3), (byte) 'D');
059    Arrays.fill(VALUE, off, (off += 5), (byte) 'E');
060    Arrays.fill(VALUE, off, (off += 8), (byte) 'F');
061    Arrays.fill(VALUE, off, (off += 13), (byte) 'G');
062    Arrays.fill(VALUE, off, (off += 21), (byte) 'H');
063    Arrays.fill(VALUE, off, (off += 34), (byte) 'I');
064    Arrays.fill(VALUE, off, (off += 55), (byte) 'J');
065    Arrays.fill(VALUE, off, (off += 89), (byte) 'K');
066    Arrays.fill(VALUE, off, (off += 144), (byte) 'L');
067    Arrays.fill(VALUE, off, (off += 233), (byte) 'M');
068    Arrays.fill(VALUE, off, (off += 377), (byte) 'N');
069    Arrays.fill(VALUE, off, (off += 610), (byte) 'O');
070    Arrays.fill(VALUE, off, (off += 987), (byte) 'P');
071    Arrays.fill(VALUE, off, (off += 1597), (byte) 'Q');
072  }
073
074  @Test
075  public void test() throws Exception {
076    testForSize(1000);
077  }
078
079  @Test
080  public void testLarge() throws Exception {
081    testForSize(1024 * 1024);
082  }
083
084  private void testForSize(int size) throws Exception {
085    TableName tableName = TableName.valueOf(getClass().getSimpleName() + "_testForSize_" + size);
086    doTest(tableName, size);
087  }
088
089  public void doTest(TableName tableName, int valueSize) throws Exception {
090    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
091    scopes.put(tableName.getName(), 0);
092    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();
093    final int total = 1000;
094    final byte[] row = Bytes.toBytes("row");
095    final byte[] family = Bytes.toBytes("family");
096    final byte[] value = new byte[valueSize];
097
098    int offset = 0;
099    while (offset + VALUE.length < value.length) {
100      System.arraycopy(VALUE, 0, value, offset, VALUE.length);
101      offset += VALUE.length;
102    }
103
104    final WALFactory wals =
105      new WALFactory(TEST_UTIL.getConfiguration(), tableName.getNameAsString());
106
107    // Write the WAL
108    final WAL wal = wals.getWAL(regionInfo);
109
110    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
111
112    for (int i = 0; i < total; i++) {
113      WALEdit kvs = new WALEdit();
114      kvs.add(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setType(Cell.Type.Put)
115        .setRow(row).setFamily(family).setQualifier(Bytes.toBytes(i)).setValue(value).build());
116      kvs.add(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
117        .setType(Cell.Type.DeleteFamily).setRow(row).setFamily(family).build());
118      wal.appendData(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName,
119        System.currentTimeMillis(), mvcc, scopes), kvs);
120      wal.sync();
121    }
122    final Path walPath = AbstractFSWALProvider.getCurrentFileName(wal);
123    wals.shutdown();
124
125    // Confirm the WAL can be read back
126    try (WALStreamReader reader = wals.createStreamReader(TEST_UTIL.getTestFileSystem(), walPath)) {
127      int count = 0;
128      WAL.Entry entry = new WAL.Entry();
129      while (reader.next(entry) != null) {
130        count++;
131        List<Cell> cells = entry.getEdit().getCells();
132        assertThat("Should be two KVs per WALEdit", cells, hasSize(2));
133        Cell putCell = cells.get(0);
134        assertEquals(Cell.Type.Put, putCell.getType());
135        assertTrue("Incorrect row", Bytes.equals(putCell.getRowArray(), putCell.getRowOffset(),
136          putCell.getRowLength(), row, 0, row.length));
137        assertTrue("Incorrect family", Bytes.equals(putCell.getFamilyArray(),
138          putCell.getFamilyOffset(), putCell.getFamilyLength(), family, 0, family.length));
139        assertTrue("Incorrect value", Bytes.equals(putCell.getValueArray(),
140          putCell.getValueOffset(), putCell.getValueLength(), value, 0, value.length));
141
142        Cell deleteCell = cells.get(1);
143        assertEquals(Cell.Type.DeleteFamily, deleteCell.getType());
144        assertTrue("Incorrect row", Bytes.equals(deleteCell.getRowArray(),
145          deleteCell.getRowOffset(), deleteCell.getRowLength(), row, 0, row.length));
146        assertTrue("Incorrect family", Bytes.equals(deleteCell.getFamilyArray(),
147          deleteCell.getFamilyOffset(), deleteCell.getFamilyLength(), family, 0, family.length));
148      }
149      assertEquals("Should have read back as many KVs as written", total, count);
150    }
151  }
152}