001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Arrays;
026import java.util.Collection;
027import java.util.Collections;
028import java.util.Comparator;
029import java.util.List;
030import java.util.NavigableSet;
031import java.util.TreeSet;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.fs.FileSystem;
034import org.apache.hadoop.fs.Path;
035import org.apache.hadoop.hbase.Cell;
036import org.apache.hadoop.hbase.CellUtil;
037import org.apache.hadoop.hbase.HBaseClassTestRule;
038import org.apache.hadoop.hbase.HBaseTestingUtil;
039import org.apache.hadoop.hbase.KeyValue;
040import org.apache.hadoop.hbase.KeyValueTestUtil;
041import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
042import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
043import org.apache.hadoop.hbase.client.Put;
044import org.apache.hadoop.hbase.client.Scan;
045import org.apache.hadoop.hbase.io.compress.Compression;
046import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter;
047import org.apache.hadoop.hbase.testclassification.RegionServerTests;
048import org.apache.hadoop.hbase.testclassification.SmallTests;
049import org.apache.hadoop.hbase.util.BloomFilterUtil;
050import org.apache.hadoop.hbase.util.Bytes;
051import org.junit.Before;
052import org.junit.ClassRule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.junit.runner.RunWith;
056import org.junit.runners.Parameterized;
057import org.junit.runners.Parameterized.Parameters;
058import org.slf4j.Logger;
059import org.slf4j.LoggerFactory;
060
061/**
062 * Test a multi-column scanner when there is a Bloom filter false-positive. This is needed for the
063 * multi-column Bloom filter optimization.
064 */
065@RunWith(Parameterized.class)
066@Category({ RegionServerTests.class, SmallTests.class })
067public class TestScanWithBloomError {
068
069  @ClassRule
070  public static final HBaseClassTestRule CLASS_RULE =
071    HBaseClassTestRule.forClass(TestScanWithBloomError.class);
072
073  private static final Logger LOG = LoggerFactory.getLogger(TestScanWithBloomError.class);
074
075  private static final String TABLE_NAME = "ScanWithBloomError";
076  private static final String FAMILY = "myCF";
077  private static final byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
078  private static final String ROW = "theRow";
079  private static final String QUALIFIER_PREFIX = "qual";
080  private static final byte[] ROW_BYTES = Bytes.toBytes(ROW);
081  private static NavigableSet<Integer> allColIds = new TreeSet<>();
082  private HRegion region;
083  private BloomType bloomType;
084  private FileSystem fs;
085  private Configuration conf;
086
087  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
088
089  @Parameters
090  public static final Collection<Object[]> parameters() {
091    List<Object[]> configurations = new ArrayList<>();
092    for (BloomType bloomType : BloomType.values()) {
093      configurations.add(new Object[] { bloomType });
094    }
095    return configurations;
096  }
097
098  public TestScanWithBloomError(BloomType bloomType) {
099    this.bloomType = bloomType;
100  }
101
102  @Before
103  public void setUp() throws IOException {
104    conf = TEST_UTIL.getConfiguration();
105    fs = FileSystem.get(conf);
106    conf.setInt(BloomFilterUtil.PREFIX_LENGTH_KEY, 10);
107  }
108
109  @Test
110  public void testThreeStoreFiles() throws IOException {
111    ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
112      .newBuilder(Bytes.toBytes(FAMILY)).setCompressionType(Compression.Algorithm.GZ)
113      .setBloomFilterType(bloomType).setMaxVersions(TestMultiColumnScanner.MAX_VERSIONS).build();
114    region = TEST_UTIL.createTestRegion(TABLE_NAME, columnFamilyDescriptor);
115    createStoreFile(new int[] { 1, 2, 6 });
116    createStoreFile(new int[] { 1, 2, 3, 7 });
117    createStoreFile(new int[] { 1, 9 });
118    scanColSet(new int[] { 1, 4, 6, 7 }, new int[] { 1, 6, 7 });
119
120    HBaseTestingUtil.closeRegionAndWAL(region);
121  }
122
123  private void scanColSet(int[] colSet, int[] expectedResultCols) throws IOException {
124    LOG.info("Scanning column set: " + Arrays.toString(colSet));
125    Scan scan = new Scan().withStartRow(ROW_BYTES).withStopRow(ROW_BYTES, true);
126    addColumnSetToScan(scan, colSet);
127    RegionScannerImpl scanner = region.getScanner(scan);
128    KeyValueHeap storeHeap = scanner.storeHeap;
129    assertEquals(0, storeHeap.getHeap().size());
130    StoreScanner storeScanner = (StoreScanner) storeHeap.getCurrentForTesting();
131    @SuppressWarnings({ "unchecked", "rawtypes" })
132    List<StoreFileScanner> scanners =
133      (List<StoreFileScanner>) (List) storeScanner.getAllScannersForTesting();
134
135    // Sort scanners by their HFile's modification time.
136    Collections.sort(scanners, new Comparator<StoreFileScanner>() {
137      @Override
138      public int compare(StoreFileScanner s1, StoreFileScanner s2) {
139        Path p1 = s1.getReader().getHFileReader().getPath();
140        Path p2 = s2.getReader().getHFileReader().getPath();
141        long t1, t2;
142        try {
143          t1 = fs.getFileStatus(p1).getModificationTime();
144          t2 = fs.getFileStatus(p2).getModificationTime();
145        } catch (IOException ex) {
146          throw new RuntimeException(ex);
147        }
148        return t1 < t2 ? -1 : t1 == t2 ? 1 : 0;
149      }
150    });
151
152    StoreFileReader lastStoreFileReader = null;
153    for (StoreFileScanner sfScanner : scanners)
154      lastStoreFileReader = sfScanner.getReader();
155
156    new HFilePrettyPrinter(conf).run(
157      new String[] { "-m", "-p", "-f", lastStoreFileReader.getHFileReader().getPath().toString() });
158
159    // Disable Bloom filter for the last store file. The disabled Bloom filter
160    // will always return "true".
161    LOG.info("Disabling Bloom filter for: " + lastStoreFileReader.getHFileReader().getName());
162    lastStoreFileReader.disableBloomFilterForTesting();
163
164    List<Cell> allResults = new ArrayList<>();
165
166    { // Limit the scope of results.
167      List<Cell> results = new ArrayList<>();
168      while (scanner.next(results) || results.size() > 0) {
169        allResults.addAll(results);
170        results.clear();
171      }
172    }
173
174    List<Integer> actualIds = new ArrayList<>();
175    for (Cell kv : allResults) {
176      String qual = Bytes.toString(CellUtil.cloneQualifier(kv));
177      assertTrue(qual.startsWith(QUALIFIER_PREFIX));
178      actualIds.add(Integer.valueOf(qual.substring(QUALIFIER_PREFIX.length())));
179    }
180    List<Integer> expectedIds = new ArrayList<>();
181    for (int expectedId : expectedResultCols)
182      expectedIds.add(expectedId);
183
184    LOG.info("Column ids returned: " + actualIds + ", expected: " + expectedIds);
185    assertEquals(expectedIds.toString(), actualIds.toString());
186  }
187
188  private void addColumnSetToScan(Scan scan, int[] colIds) {
189    for (int colId : colIds) {
190      scan.addColumn(FAMILY_BYTES, Bytes.toBytes(qualFromId(colId)));
191    }
192  }
193
194  private String qualFromId(int colId) {
195    return QUALIFIER_PREFIX + colId;
196  }
197
198  private void createStoreFile(int[] colIds) throws IOException {
199    Put p = new Put(ROW_BYTES);
200    for (int colId : colIds) {
201      long ts = Long.MAX_VALUE;
202      String qual = qualFromId(colId);
203      allColIds.add(colId);
204      KeyValue kv = KeyValueTestUtil.create(ROW, FAMILY, qual, ts,
205        TestMultiColumnScanner.createValue(ROW, qual, ts));
206      p.add(kv);
207    }
208    region.put(p);
209    region.flush(true);
210  }
211
212}