001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.regionserver.Store.PRIORITY_USER;
021
022import java.io.IOException;
023import java.security.Key;
024import java.util.ArrayList;
025import java.util.Collection;
026import java.util.Collections;
027import java.util.Date;
028import java.util.Iterator;
029import java.util.List;
030import java.util.NavigableSet;
031import java.util.Optional;
032import java.util.concurrent.ConcurrentSkipListSet;
033import javax.crypto.spec.SecretKeySpec;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.fs.FileSystem;
036import org.apache.hadoop.fs.Path;
037import org.apache.hadoop.hbase.ArrayBackedTag;
038import org.apache.hadoop.hbase.Cell;
039import org.apache.hadoop.hbase.CellComparatorImpl;
040import org.apache.hadoop.hbase.CellUtil;
041import org.apache.hadoop.hbase.ExtendedCell;
042import org.apache.hadoop.hbase.HBaseClassTestRule;
043import org.apache.hadoop.hbase.HBaseConfiguration;
044import org.apache.hadoop.hbase.HBaseTestingUtil;
045import org.apache.hadoop.hbase.HConstants;
046import org.apache.hadoop.hbase.KeyValue;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.Tag;
049import org.apache.hadoop.hbase.TagType;
050import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
051import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
052import org.apache.hadoop.hbase.client.Get;
053import org.apache.hadoop.hbase.client.RegionInfo;
054import org.apache.hadoop.hbase.client.RegionInfoBuilder;
055import org.apache.hadoop.hbase.client.Scan;
056import org.apache.hadoop.hbase.client.TableDescriptor;
057import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
058import org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider;
059import org.apache.hadoop.hbase.io.crypto.aes.AES;
060import org.apache.hadoop.hbase.io.hfile.HFile;
061import org.apache.hadoop.hbase.mob.MobConstants;
062import org.apache.hadoop.hbase.mob.MobFileCache;
063import org.apache.hadoop.hbase.mob.MobUtils;
064import org.apache.hadoop.hbase.monitoring.MonitoredTask;
065import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
066import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
067import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
068import org.apache.hadoop.hbase.security.EncryptionUtil;
069import org.apache.hadoop.hbase.security.User;
070import org.apache.hadoop.hbase.testclassification.MediumTests;
071import org.apache.hadoop.hbase.util.Bytes;
072import org.apache.hadoop.hbase.util.CommonFSUtils;
073import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
074import org.apache.hadoop.hbase.wal.WALFactory;
075import org.junit.Assert;
076import org.junit.Before;
077import org.junit.ClassRule;
078import org.junit.Rule;
079import org.junit.Test;
080import org.junit.experimental.categories.Category;
081import org.junit.rules.TestName;
082import org.mockito.Mockito;
083import org.slf4j.Logger;
084import org.slf4j.LoggerFactory;
085
086@Category(MediumTests.class)
087public class TestHMobStore {
088
089  @ClassRule
090  public static final HBaseClassTestRule CLASS_RULE =
091    HBaseClassTestRule.forClass(TestHMobStore.class);
092
093  public static final Logger LOG = LoggerFactory.getLogger(TestHMobStore.class);
094  @Rule
095  public TestName name = new TestName();
096
097  private HMobStore store;
098  private HRegion region;
099  private FileSystem fs;
100  private byte[] table = Bytes.toBytes("table");
101  private byte[] family = Bytes.toBytes("family");
102  private byte[] row = Bytes.toBytes("row");
103  private byte[] row2 = Bytes.toBytes("row2");
104  private byte[] qf1 = Bytes.toBytes("qf1");
105  private byte[] qf2 = Bytes.toBytes("qf2");
106  private byte[] qf3 = Bytes.toBytes("qf3");
107  private byte[] qf4 = Bytes.toBytes("qf4");
108  private byte[] qf5 = Bytes.toBytes("qf5");
109  private byte[] qf6 = Bytes.toBytes("qf6");
110  private byte[] value = Bytes.toBytes("value");
111  private byte[] value2 = Bytes.toBytes("value2");
112  private Path mobFilePath;
113  private Date currentDate = new Date();
114  private ExtendedCell seekKey1;
115  private ExtendedCell seekKey2;
116  private ExtendedCell seekKey3;
117  private NavigableSet<byte[]> qualifiers = new ConcurrentSkipListSet<>(Bytes.BYTES_COMPARATOR);
118  private List<ExtendedCell> expected = new ArrayList<>();
119  private long id = EnvironmentEdgeManager.currentTime();
120  private Get get = new Get(row);
121  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
122  private final String DIR = TEST_UTIL.getDataTestDir("TestHMobStore").toString();
123
124  /**
125   * Setup
126   */
127  @Before
128  public void setUp() throws Exception {
129    qualifiers.add(qf1);
130    qualifiers.add(qf3);
131    qualifiers.add(qf5);
132
133    Iterator<byte[]> iter = qualifiers.iterator();
134    while (iter.hasNext()) {
135      byte[] next = iter.next();
136      expected.add(new KeyValue(row, family, next, 1, value));
137      get.addColumn(family, next);
138      get.readAllVersions();
139    }
140  }
141
142  private void init(String methodName, Configuration conf, boolean testStore) throws IOException {
143    ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(family)
144      .setMobEnabled(true).setMobThreshold(3L).setMaxVersions(4).build();
145    init(methodName, conf, cfd, testStore);
146  }
147
148  private void init(String methodName, Configuration conf, ColumnFamilyDescriptor cfd,
149    boolean testStore) throws IOException {
150    TableDescriptor td =
151      TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setColumnFamily(cfd).build();
152
153    // Setting up tje Region and Store
154    Path basedir = new Path(DIR + methodName);
155    Path tableDir = CommonFSUtils.getTableDir(basedir, td.getTableName());
156    String logName = "logs";
157    Path logdir = new Path(basedir, logName);
158    FileSystem fs = FileSystem.get(conf);
159    fs.delete(logdir, true);
160
161    RegionInfo info = RegionInfoBuilder.newBuilder(td.getTableName()).build();
162    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
163      MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
164    final Configuration walConf = new Configuration(conf);
165    CommonFSUtils.setRootDir(walConf, basedir);
166    final WALFactory wals = new WALFactory(walConf, methodName);
167    region = new HRegion(tableDir, wals.getWAL(info), fs, conf, info, td, null);
168    region.setMobFileCache(new MobFileCache(conf));
169    store = new HMobStore(region, cfd, conf, false);
170    if (testStore) {
171      init(conf, cfd);
172    }
173  }
174
175  private void init(Configuration conf, ColumnFamilyDescriptor cfd) throws IOException {
176    Path basedir = CommonFSUtils.getRootDir(conf);
177    fs = FileSystem.get(conf);
178    Path homePath =
179      new Path(basedir, Bytes.toString(family) + Path.SEPARATOR + Bytes.toString(family));
180    fs.mkdirs(homePath);
181
182    KeyValue key1 = new KeyValue(row, family, qf1, 1, value);
183    KeyValue key2 = new KeyValue(row, family, qf2, 1, value);
184    KeyValue key3 = new KeyValue(row2, family, qf3, 1, value2);
185    KeyValue[] keys = new KeyValue[] { key1, key2, key3 };
186    int maxKeyCount = keys.length;
187    StoreFileWriter mobWriter = store.createWriterInTmp(currentDate, maxKeyCount,
188      cfd.getCompactionCompressionType(), region.getRegionInfo().getStartKey(), false);
189    mobFilePath = mobWriter.getPath();
190
191    mobWriter.append(key1);
192    mobWriter.append(key2);
193    mobWriter.append(key3);
194    mobWriter.close();
195
196    String targetPathName = MobUtils.formatDate(currentDate);
197    byte[] referenceValue = Bytes.toBytes(targetPathName + Path.SEPARATOR + mobFilePath.getName());
198    Tag tableNameTag =
199      new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE, store.getTableName().getName());
200    KeyValue kv1 = new KeyValue(row, family, qf1, Long.MAX_VALUE, referenceValue);
201    KeyValue kv2 = new KeyValue(row, family, qf2, Long.MAX_VALUE, referenceValue);
202    KeyValue kv3 = new KeyValue(row2, family, qf3, Long.MAX_VALUE, referenceValue);
203    seekKey1 = MobUtils.createMobRefCell(kv1, referenceValue, tableNameTag);
204    seekKey2 = MobUtils.createMobRefCell(kv2, referenceValue, tableNameTag);
205    seekKey3 = MobUtils.createMobRefCell(kv3, referenceValue, tableNameTag);
206  }
207
208  /**
209   * Getting data from memstore
210   */
211  @Test
212  public void testGetFromMemStore() throws IOException {
213    final Configuration conf = HBaseConfiguration.create();
214    init(name.getMethodName(), conf, false);
215
216    // Put data in memstore
217    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
218    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
219    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
220    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
221    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
222    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
223
224    Scan scan = new Scan(get);
225    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
226      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0);
227
228    List<Cell> results = new ArrayList<>();
229    scanner.next(results);
230    Collections.sort(results, CellComparatorImpl.COMPARATOR);
231    scanner.close();
232
233    // Compare
234    Assert.assertEquals(expected.size(), results.size());
235    for (int i = 0; i < results.size(); i++) {
236      // Verify the values
237      Assert.assertEquals(expected.get(i), results.get(i));
238    }
239  }
240
241  /**
242   * Getting MOB data from files
243   */
244  @Test
245  public void testGetFromFiles() throws IOException {
246    final Configuration conf = TEST_UTIL.getConfiguration();
247    init(name.getMethodName(), conf, false);
248
249    // Put data in memstore
250    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
251    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
252    // flush
253    flush(1);
254
255    // Add more data
256    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
257    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
258    // flush
259    flush(2);
260
261    // Add more data
262    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
263    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
264    // flush
265    flush(3);
266
267    Scan scan = new Scan(get);
268    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
269      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0);
270
271    List<Cell> results = new ArrayList<>();
272    scanner.next(results);
273    Collections.sort(results, CellComparatorImpl.COMPARATOR);
274    scanner.close();
275
276    // Compare
277    Assert.assertEquals(expected.size(), results.size());
278    for (int i = 0; i < results.size(); i++) {
279      Assert.assertEquals(expected.get(i), results.get(i));
280    }
281  }
282
283  /**
284   * Getting the reference data from files
285   */
286  @Test
287  public void testGetReferencesFromFiles() throws IOException {
288    final Configuration conf = HBaseConfiguration.create();
289    init(name.getMethodName(), conf, false);
290
291    // Put data in memstore
292    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
293    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
294    // flush
295    flush(1);
296
297    // Add more data
298    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
299    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
300    // flush
301    flush(2);
302
303    // Add more data
304    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
305    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
306    // flush
307    flush(3);
308
309    Scan scan = new Scan(get);
310    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
311    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
312      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0);
313
314    List<ExtendedCell> results = new ArrayList<>();
315    scanner.next(results);
316    Collections.sort(results, CellComparatorImpl.COMPARATOR);
317    scanner.close();
318
319    // Compare
320    Assert.assertEquals(expected.size(), results.size());
321    for (int i = 0; i < results.size(); i++) {
322      ExtendedCell cell = results.get(i);
323      Assert.assertTrue(MobUtils.isMobReferenceCell(cell));
324    }
325  }
326
327  /**
328   * Getting data from memstore and files
329   */
330  @Test
331  public void testGetFromMemStoreAndFiles() throws IOException {
332
333    final Configuration conf = HBaseConfiguration.create();
334
335    init(name.getMethodName(), conf, false);
336
337    // Put data in memstore
338    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
339    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
340    // flush
341    flush(1);
342
343    // Add more data
344    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
345    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
346    // flush
347    flush(2);
348
349    // Add more data
350    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
351    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
352
353    Scan scan = new Scan(get);
354    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
355      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0);
356
357    List<Cell> results = new ArrayList<>();
358    scanner.next(results);
359    Collections.sort(results, CellComparatorImpl.COMPARATOR);
360    scanner.close();
361
362    // Compare
363    Assert.assertEquals(expected.size(), results.size());
364    for (int i = 0; i < results.size(); i++) {
365      Assert.assertEquals(expected.get(i), results.get(i));
366    }
367  }
368
369  /**
370   * Getting data from memstore and files
371   */
372  @Test
373  public void testMobCellSizeThreshold() throws IOException {
374    final Configuration conf = HBaseConfiguration.create();
375    ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(family)
376      .setMobEnabled(true).setMobThreshold(100).setMaxVersions(4).build();
377    init(name.getMethodName(), conf, cfd, false);
378
379    // Put data in memstore
380    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
381    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
382    // flush
383    flush(1);
384
385    // Add more data
386    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
387    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
388    // flush
389    flush(2);
390
391    // Add more data
392    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
393    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
394    // flush
395    flush(3);
396
397    Scan scan = new Scan(get);
398    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
399    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
400      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0);
401
402    List<ExtendedCell> results = new ArrayList<>();
403    scanner.next(results);
404    Collections.sort(results, CellComparatorImpl.COMPARATOR);
405    scanner.close();
406
407    // Compare
408    Assert.assertEquals(expected.size(), results.size());
409    for (int i = 0; i < results.size(); i++) {
410      ExtendedCell cell = results.get(i);
411      // this is not mob reference cell.
412      Assert.assertFalse(MobUtils.isMobReferenceCell(cell));
413      Assert.assertEquals(expected.get(i), results.get(i));
414      Assert.assertEquals(100, store.getColumnFamilyDescriptor().getMobThreshold());
415    }
416  }
417
418  @Test
419  public void testCommitFile() throws Exception {
420    final Configuration conf = HBaseConfiguration.create();
421    init(name.getMethodName(), conf, true);
422    String targetPathName = MobUtils.formatDate(new Date());
423    Path targetPath =
424      new Path(store.getPath(), (targetPathName + Path.SEPARATOR + mobFilePath.getName()));
425    fs.delete(targetPath, true);
426    Assert.assertFalse(fs.exists(targetPath));
427    // commit file
428    store.commitFile(mobFilePath, targetPath);
429    Assert.assertTrue(fs.exists(targetPath));
430  }
431
432  @Test
433  public void testResolve() throws Exception {
434    final Configuration conf = HBaseConfiguration.create();
435    init(name.getMethodName(), conf, true);
436    String targetPathName = MobUtils.formatDate(currentDate);
437    Path targetPath = new Path(store.getPath(), targetPathName);
438    store.commitFile(mobFilePath, targetPath);
439    // resolve
440    Cell resultCell1 = store.resolve(seekKey1, false).getCell();
441    Cell resultCell2 = store.resolve(seekKey2, false).getCell();
442    Cell resultCell3 = store.resolve(seekKey3, false).getCell();
443    // compare
444    Assert.assertEquals(Bytes.toString(value), Bytes.toString(CellUtil.cloneValue(resultCell1)));
445    Assert.assertEquals(Bytes.toString(value), Bytes.toString(CellUtil.cloneValue(resultCell2)));
446    Assert.assertEquals(Bytes.toString(value2), Bytes.toString(CellUtil.cloneValue(resultCell3)));
447  }
448
449  /**
450   * Flush the memstore
451   */
452  private void flush(int storeFilesSize) throws IOException {
453    flushStore(store, id++);
454    Assert.assertEquals(storeFilesSize, this.store.getStorefiles().size());
455    Assert.assertEquals(0, ((AbstractMemStore) this.store.memstore).getActive().getCellsCount());
456  }
457
458  /**
459   * Flush the memstore
460   */
461  private static void flushStore(HMobStore store, long id) throws IOException {
462    StoreFlushContext storeFlushCtx = store.createFlushContext(id, FlushLifeCycleTracker.DUMMY);
463    storeFlushCtx.prepare();
464    storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
465    storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
466  }
467
468  @Test
469  public void testMOBStoreEncryption() throws Exception {
470    final Configuration conf = TEST_UTIL.getConfiguration();
471
472    conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, MockAesKeyProvider.class.getName());
473    conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
474    byte[] keyBytes = new byte[AES.KEY_LENGTH];
475    Bytes.secureRandom(keyBytes);
476    String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
477    Key cfKey = new SecretKeySpec(keyBytes, algorithm);
478
479    ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(family)
480      .setMobEnabled(true).setMobThreshold(100).setMaxVersions(4).setEncryptionType(algorithm)
481      .setEncryptionKey(EncryptionUtil.wrapKey(conf,
482        conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()),
483        cfKey))
484      .build();
485    init(name.getMethodName(), conf, cfd, false);
486
487    this.store.add(new KeyValue(row, family, qf1, 1, value), null);
488    this.store.add(new KeyValue(row, family, qf2, 1, value), null);
489    this.store.add(new KeyValue(row, family, qf3, 1, value), null);
490    flush(1);
491
492    this.store.add(new KeyValue(row, family, qf4, 1, value), null);
493    this.store.add(new KeyValue(row, family, qf5, 1, value), null);
494    this.store.add(new KeyValue(row, family, qf6, 1, value), null);
495    flush(2);
496
497    Collection<HStoreFile> storefiles = this.store.getStorefiles();
498    checkMobHFileEncrytption(storefiles);
499
500    // Scan the values
501    Scan scan = new Scan(get);
502    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
503      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0);
504
505    List<Cell> results = new ArrayList<>();
506    scanner.next(results);
507    Collections.sort(results, CellComparatorImpl.COMPARATOR);
508    scanner.close();
509    Assert.assertEquals(expected.size(), results.size());
510    for (int i = 0; i < results.size(); i++) {
511      Assert.assertEquals(expected.get(i), results.get(i));
512    }
513
514    // Trigger major compaction
515    this.store.triggerMajorCompaction();
516    Optional<CompactionContext> requestCompaction =
517      this.store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null);
518    this.store.compact(requestCompaction.get(), NoLimitThroughputController.INSTANCE, null);
519    Assert.assertEquals(1, this.store.getStorefiles().size());
520
521    // Check encryption after compaction
522    checkMobHFileEncrytption(this.store.getStorefiles());
523  }
524
525  private void checkMobHFileEncrytption(Collection<HStoreFile> storefiles) {
526    HStoreFile storeFile = storefiles.iterator().next();
527    HFile.Reader reader = storeFile.getReader().getHFileReader();
528    byte[] encryptionKey = reader.getTrailer().getEncryptionKey();
529    Assert.assertTrue(null != encryptionKey);
530    Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName()
531      .equals(HConstants.CIPHER_AES));
532  }
533
534}