Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.io.ByteArrayInputStream;
026import java.io.ByteArrayOutputStream;
027import java.io.DataInputStream;
028import java.io.DataOutputStream;
029import java.io.IOException;
030import java.util.List;
031import java.util.Map;
032import java.util.NavigableSet;
033import java.util.Set;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.Get;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.client.RegionInfoBuilder;
039import org.apache.hadoop.hbase.client.Scan;
040import org.apache.hadoop.hbase.client.TableDescriptor;
041import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
042import org.apache.hadoop.hbase.filter.BinaryComparator;
043import org.apache.hadoop.hbase.filter.Filter;
044import org.apache.hadoop.hbase.filter.PrefixFilter;
045import org.apache.hadoop.hbase.filter.RowFilter;
046import org.apache.hadoop.hbase.io.TimeRange;
047import org.apache.hadoop.hbase.testclassification.MiscTests;
048import org.apache.hadoop.hbase.testclassification.SmallTests;
049import org.apache.hadoop.hbase.util.ByteBufferUtils;
050import org.apache.hadoop.hbase.util.Bytes;
051import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
052import org.apache.hadoop.io.DataInputBuffer;
053import org.junit.ClassRule;
054import org.junit.Test;
055import org.junit.experimental.categories.Category;
056
057import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
058import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
059
060/**
061 * Test HBase Writables serializations
062 */
063@Category({ MiscTests.class, SmallTests.class })
064public class TestSerialization {
065
066  @ClassRule
067  public static final HBaseClassTestRule CLASS_RULE =
068    HBaseClassTestRule.forClass(TestSerialization.class);
069
070  @Test
071  public void testKeyValue() throws Exception {
072    final String name = "testKeyValue2";
073    byte[] row = Bytes.toBytes(name);
074    byte[] fam = Bytes.toBytes("fam");
075    byte[] qf = Bytes.toBytes("qf");
076    long ts = EnvironmentEdgeManager.currentTime();
077    byte[] val = Bytes.toBytes("val");
078    KeyValue kv = new KeyValue(row, fam, qf, ts, val);
079    ByteArrayOutputStream baos = new ByteArrayOutputStream();
080    DataOutputStream dos = new DataOutputStream(baos);
081    KeyValueUtil.write(kv, dos);
082    dos.close();
083    byte[] mb = baos.toByteArray();
084    ByteArrayInputStream bais = new ByteArrayInputStream(mb);
085    DataInputStream dis = new DataInputStream(bais);
086    KeyValue deserializedKv = KeyValueUtil.create(dis);
087    assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer()));
088    assertEquals(kv.getOffset(), deserializedKv.getOffset());
089    assertEquals(kv.getLength(), deserializedKv.getLength());
090  }
091
092  @Test
093  public void testCreateKeyValueInvalidNegativeLength() {
094
095    KeyValue kv_0 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), // 51 bytes
096      Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my12345"));
097
098    KeyValue kv_1 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), // 49 bytes
099      Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my123"));
100
101    ByteArrayOutputStream baos = new ByteArrayOutputStream();
102    DataOutputStream dos = new DataOutputStream(baos);
103
104    long l = 0;
105    try {
106      ByteBufferUtils.putInt(dos, kv_0.getSerializedSize(false));
107      l = (long) kv_0.write(dos, false) + Bytes.SIZEOF_INT;
108      ByteBufferUtils.putInt(dos, kv_1.getSerializedSize(false));
109      l += (long) kv_1.write(dos, false) + Bytes.SIZEOF_INT;
110      assertEquals(100L, l);
111    } catch (IOException e) {
112      fail("Unexpected IOException" + e.getMessage());
113    }
114
115    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
116    DataInputStream dis = new DataInputStream(bais);
117
118    try {
119      KeyValueUtil.create(dis);
120      assertTrue(kv_0.equals(kv_1));
121    } catch (Exception e) {
122      fail("Unexpected Exception" + e.getMessage());
123    }
124
125    // length -1
126    try {
127      // even if we have a good kv now in dis we will just pass length with -1 for simplicity
128      KeyValueUtil.create(-1, dis);
129      fail("Expected corrupt stream");
130    } catch (Exception e) {
131      assertEquals("Failed read -1 bytes, stream corrupt?", e.getMessage());
132    }
133
134  }
135
136  @Test
137  public void testCompareFilter() throws Exception {
138    Filter f =
139      new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2")));
140    byte[] bytes = f.toByteArray();
141    Filter ff = RowFilter.parseFrom(bytes);
142    assertNotNull(ff);
143  }
144
145  @Test
146  public void testTableDescriptor() throws Exception {
147    final String name = "testTableDescriptor";
148    TableDescriptor htd = createTableDescriptor(name);
149    byte[] mb = TableDescriptorBuilder.toByteArray(htd);
150    TableDescriptor deserializedHtd = TableDescriptorBuilder.parseFrom(mb);
151    assertEquals(htd.getTableName(), deserializedHtd.getTableName());
152  }
153
154  /**
155   * Test RegionInfo serialization
156   */
157  @Test
158  public void testRegionInfo() throws Exception {
159    RegionInfo hri = createRandomRegion("testRegionInfo");
160
161    // test toByteArray()
162    byte[] hrib = RegionInfo.toByteArray(hri);
163    RegionInfo deserializedHri = RegionInfo.parseFrom(hrib);
164    assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
165    assertEquals(hri, deserializedHri);
166
167    // test toDelimitedByteArray()
168    hrib = RegionInfo.toDelimitedByteArray(hri);
169    DataInputBuffer buf = new DataInputBuffer();
170    try {
171      buf.reset(hrib, hrib.length);
172      deserializedHri = RegionInfo.parseFrom(buf);
173      assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
174      assertEquals(hri, deserializedHri);
175    } finally {
176      buf.close();
177    }
178  }
179
180  @Test
181  public void testRegionInfos() throws Exception {
182    RegionInfo hri = createRandomRegion("testRegionInfos");
183    byte[] triple = RegionInfo.toDelimitedByteArray(hri, hri, hri);
184    List<RegionInfo> regions = RegionInfo.parseDelimitedFrom(triple, 0, triple.length);
185    assertTrue(regions.size() == 3);
186    assertTrue(regions.get(0).equals(regions.get(1)));
187    assertTrue(regions.get(0).equals(regions.get(2)));
188  }
189
190  private RegionInfo createRandomRegion(final String name) {
191    TableDescriptorBuilder tableDescriptorBuilder =
192      TableDescriptorBuilder.newBuilder(TableName.valueOf(name));
193    String[] families = new String[] { "info", "anchor" };
194    for (int i = 0; i < families.length; i++) {
195      ColumnFamilyDescriptor columnFamilyDescriptor =
196        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(families[i])).build();
197      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
198    }
199    TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
200    return RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
201  }
202
203  @Test
204  public void testGet() throws Exception {
205    byte[] row = Bytes.toBytes("row");
206    byte[] fam = Bytes.toBytes("fam");
207    byte[] qf1 = Bytes.toBytes("qf1");
208    long ts = EnvironmentEdgeManager.currentTime();
209    int maxVersions = 2;
210
211    Get get = new Get(row);
212    get.addColumn(fam, qf1);
213    get.setTimeRange(ts, ts + 1);
214    get.readVersions(maxVersions);
215
216    ClientProtos.Get getProto = ProtobufUtil.toGet(get);
217    Get desGet = ProtobufUtil.toGet(getProto);
218
219    assertTrue(Bytes.equals(get.getRow(), desGet.getRow()));
220    Set<byte[]> set = null;
221    Set<byte[]> desSet = null;
222
223    for (Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()) {
224      assertTrue(desGet.getFamilyMap().containsKey(entry.getKey()));
225      set = entry.getValue();
226      desSet = desGet.getFamilyMap().get(entry.getKey());
227      for (byte[] qualifier : set) {
228        assertTrue(desSet.contains(qualifier));
229      }
230    }
231
232    assertEquals(get.getMaxVersions(), desGet.getMaxVersions());
233    TimeRange tr = get.getTimeRange();
234    TimeRange desTr = desGet.getTimeRange();
235    assertEquals(tr.getMax(), desTr.getMax());
236    assertEquals(tr.getMin(), desTr.getMin());
237  }
238
239  @Test
240  public void testScan() throws Exception {
241
242    byte[] startRow = Bytes.toBytes("startRow");
243    byte[] stopRow = Bytes.toBytes("stopRow");
244    byte[] fam = Bytes.toBytes("fam");
245    byte[] qf1 = Bytes.toBytes("qf1");
246    long ts = EnvironmentEdgeManager.currentTime();
247    int maxVersions = 2;
248
249    Scan scan = new Scan().withStartRow(startRow).withStopRow(stopRow);
250    scan.addColumn(fam, qf1);
251    scan.setTimeRange(ts, ts + 1);
252    scan.readVersions(maxVersions);
253
254    ClientProtos.Scan scanProto = ProtobufUtil.toScan(scan);
255    Scan desScan = ProtobufUtil.toScan(scanProto);
256
257    assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow()));
258    assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow()));
259    assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks());
260    Set<byte[]> set = null;
261    Set<byte[]> desSet = null;
262
263    for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
264      assertTrue(desScan.getFamilyMap().containsKey(entry.getKey()));
265      set = entry.getValue();
266      desSet = desScan.getFamilyMap().get(entry.getKey());
267      for (byte[] column : set) {
268        assertTrue(desSet.contains(column));
269      }
270
271      // Test filters are serialized properly.
272      scan = new Scan().withStartRow(startRow);
273      final String name = "testScan";
274      byte[] prefix = Bytes.toBytes(name);
275      scan.setFilter(new PrefixFilter(prefix));
276      scanProto = ProtobufUtil.toScan(scan);
277      desScan = ProtobufUtil.toScan(scanProto);
278      Filter f = desScan.getFilter();
279      assertTrue(f instanceof PrefixFilter);
280    }
281
282    assertEquals(scan.getMaxVersions(), desScan.getMaxVersions());
283    TimeRange tr = scan.getTimeRange();
284    TimeRange desTr = desScan.getTimeRange();
285    assertEquals(tr.getMax(), desTr.getMax());
286    assertEquals(tr.getMin(), desTr.getMin());
287  }
288
289  protected static final int MAXVERSIONS = 3;
290  protected final static byte[] fam1 = Bytes.toBytes("colfamily1");
291  protected final static byte[] fam2 = Bytes.toBytes("colfamily2");
292  protected final static byte[] fam3 = Bytes.toBytes("colfamily3");
293  protected static final byte[][] COLUMNS = { fam1, fam2, fam3 };
294
295  /**
296   * Create a table of name <code>name</code> with {@link #COLUMNS} for families.
297   * @param name Name to give table.
298   * @return Column descriptor.
299   */
300  protected TableDescriptor createTableDescriptor(final String name) {
301    return createTableDescriptor(name, MAXVERSIONS);
302  }
303
304  /**
305   * Create a table of name <code>name</code> with {@link #COLUMNS} for families.
306   * @param name     Name to give table.
307   * @param versions How many versions to allow per column.
308   * @return Column descriptor.
309   */
310  protected TableDescriptor createTableDescriptor(final String name, final int versions) {
311    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name));
312    builder
313      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1).setMaxVersions(versions)
314        .setBlockCacheEnabled(false).build())
315      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam2).setMaxVersions(versions)
316        .setBlockCacheEnabled(false).build())
317      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam3).setMaxVersions(versions)
318        .setBlockCacheEnabled(false).build());
319    return builder.build();
320  }
321}