001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Collection;
026import java.util.HashMap;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Set;
030import org.apache.hadoop.hbase.Cell;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.KeyValue;
034import org.apache.hadoop.hbase.KeyValueTestUtil;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
037import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
038import org.apache.hadoop.hbase.client.Durability;
039import org.apache.hadoop.hbase.client.Put;
040import org.apache.hadoop.hbase.client.RegionInfo;
041import org.apache.hadoop.hbase.client.RegionInfoBuilder;
042import org.apache.hadoop.hbase.client.Scan;
043import org.apache.hadoop.hbase.client.TableDescriptor;
044import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
045import org.apache.hadoop.hbase.testclassification.MediumTests;
046import org.apache.hadoop.hbase.testclassification.RegionServerTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.junit.ClassRule;
049import org.junit.Rule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052import org.junit.rules.TestName;
053import org.slf4j.Logger;
054import org.slf4j.LoggerFactory;
055
056@Category({ RegionServerTests.class, MediumTests.class })
057public class TestColumnSeeking {
058
059  @ClassRule
060  public static final HBaseClassTestRule CLASS_RULE =
061    HBaseClassTestRule.forClass(TestColumnSeeking.class);
062
063  @Rule
064  public TestName name = new TestName();
065  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
066
067  private static final Logger LOG = LoggerFactory.getLogger(TestColumnSeeking.class);
068
069  @SuppressWarnings("unchecked")
070  @Test
071  public void testDuplicateVersions() throws IOException {
072    String family = "Family";
073    byte[] familyBytes = Bytes.toBytes("Family");
074    TableName table = TableName.valueOf(name.getMethodName());
075
076    ColumnFamilyDescriptor familyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(familyBytes)
077      .setMaxVersions(1000).setMaxVersions(3).build();
078    TableDescriptor tableDescriptor =
079      TableDescriptorBuilder.newBuilder(table).setColumnFamily(familyDescriptor).build();
080    RegionInfo info = RegionInfoBuilder.newBuilder(table).build();
081    // Set this so that the archiver writes to the temp dir as well.
082    HRegion region = TEST_UTIL.createLocalHRegion(info, tableDescriptor);
083    try {
084      List<String> rows = generateRandomWords(10, "row");
085      List<String> allColumns = generateRandomWords(10, "column");
086      List<String> values = generateRandomWords(100, "value");
087
088      long maxTimestamp = 2;
089      double selectPercent = 0.5;
090      int numberOfTests = 5;
091      double flushPercentage = 0.2;
092      double minorPercentage = 0.2;
093      double majorPercentage = 0.2;
094      double putPercentage = 0.2;
095
096      HashMap<String, KeyValue> allKVMap = new HashMap<>();
097
098      HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
099      ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
100
101      for (int i = 0; i < numberOfTests; i++) {
102        kvMaps[i] = new HashMap<>();
103        columnLists[i] = new ArrayList<>();
104        for (String column : allColumns) {
105          if (Math.random() < selectPercent) {
106            columnLists[i].add(column);
107          }
108        }
109      }
110
111      for (String value : values) {
112        for (String row : rows) {
113          Put p = new Put(Bytes.toBytes(row));
114          p.setDurability(Durability.SKIP_WAL);
115          for (String column : allColumns) {
116            for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
117              KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, value);
118              if (Math.random() < putPercentage) {
119                p.add(kv);
120                allKVMap.put(kv.getKeyString(), kv);
121                for (int i = 0; i < numberOfTests; i++) {
122                  if (columnLists[i].contains(column)) {
123                    kvMaps[i].put(kv.getKeyString(), kv);
124                  }
125                }
126              }
127            }
128          }
129          region.put(p);
130          if (Math.random() < flushPercentage) {
131            LOG.info("Flushing... ");
132            region.flush(true);
133          }
134
135          if (Math.random() < minorPercentage) {
136            LOG.info("Minor compacting... ");
137            region.compact(false);
138          }
139
140          if (Math.random() < majorPercentage) {
141            LOG.info("Major compacting... ");
142            region.compact(true);
143          }
144        }
145      }
146
147      for (int i = 0; i < numberOfTests + 1; i++) {
148        Collection<KeyValue> kvSet;
149        Scan scan = new Scan();
150        scan.readAllVersions();
151        if (i < numberOfTests) {
152          if (columnLists[i].isEmpty()) continue; // HBASE-7700
153          kvSet = kvMaps[i].values();
154          for (String column : columnLists[i]) {
155            scan.addColumn(familyBytes, Bytes.toBytes(column));
156          }
157          LOG.info("ExplicitColumns scanner");
158          LOG.info("Columns: " + columnLists[i].size() + "  Keys: " + kvSet.size());
159        } else {
160          kvSet = allKVMap.values();
161          LOG.info("Wildcard scanner");
162          LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
163
164        }
165        InternalScanner scanner = region.getScanner(scan);
166        List<Cell> results = new ArrayList<>();
167        while (scanner.next(results))
168          ;
169        assertEquals(kvSet.size(), results.size());
170        assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
171      }
172    } finally {
173      HBaseTestingUtil.closeRegionAndWAL(region);
174    }
175
176    HBaseTestingUtil.closeRegionAndWAL(region);
177  }
178
179  @SuppressWarnings("unchecked")
180  @Test
181  public void testReseeking() throws IOException {
182    String family = "Family";
183    byte[] familyBytes = Bytes.toBytes("Family");
184    TableName table = TableName.valueOf(name.getMethodName());
185
186    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(table);
187    ColumnFamilyDescriptor columnFamilyDescriptor =
188      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).setMaxVersions(3).build();
189    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
190
191    RegionInfo info = RegionInfoBuilder.newBuilder(table).build();
192    HRegion region = TEST_UTIL.createLocalHRegion(info, tableDescriptorBuilder.build());
193
194    List<String> rows = generateRandomWords(10, "row");
195    List<String> allColumns = generateRandomWords(100, "column");
196
197    long maxTimestamp = 2;
198    double selectPercent = 0.5;
199    int numberOfTests = 5;
200    double flushPercentage = 0.2;
201    double minorPercentage = 0.2;
202    double majorPercentage = 0.2;
203    double putPercentage = 0.2;
204
205    HashMap<String, KeyValue> allKVMap = new HashMap<>();
206
207    HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
208    ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
209    String valueString = "Value";
210
211    for (int i = 0; i < numberOfTests; i++) {
212      kvMaps[i] = new HashMap<>();
213      columnLists[i] = new ArrayList<>();
214      for (String column : allColumns) {
215        if (Math.random() < selectPercent) {
216          columnLists[i].add(column);
217        }
218      }
219    }
220
221    for (String row : rows) {
222      Put p = new Put(Bytes.toBytes(row));
223      p.setDurability(Durability.SKIP_WAL);
224      for (String column : allColumns) {
225        for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
226          KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString);
227          if (Math.random() < putPercentage) {
228            p.add(kv);
229            allKVMap.put(kv.getKeyString(), kv);
230            for (int i = 0; i < numberOfTests; i++) {
231              if (columnLists[i].contains(column)) {
232                kvMaps[i].put(kv.getKeyString(), kv);
233              }
234            }
235          }
236
237        }
238      }
239      region.put(p);
240      if (Math.random() < flushPercentage) {
241        LOG.info("Flushing... ");
242        region.flush(true);
243      }
244
245      if (Math.random() < minorPercentage) {
246        LOG.info("Minor compacting... ");
247        region.compact(false);
248      }
249
250      if (Math.random() < majorPercentage) {
251        LOG.info("Major compacting... ");
252        region.compact(true);
253      }
254    }
255
256    for (int i = 0; i < numberOfTests + 1; i++) {
257      Collection<KeyValue> kvSet;
258      Scan scan = new Scan();
259      scan.readAllVersions();
260      if (i < numberOfTests) {
261        if (columnLists[i].isEmpty()) continue; // HBASE-7700
262        kvSet = kvMaps[i].values();
263        for (String column : columnLists[i]) {
264          scan.addColumn(familyBytes, Bytes.toBytes(column));
265        }
266        LOG.info("ExplicitColumns scanner");
267        LOG.info("Columns: " + columnLists[i].size() + "  Keys: " + kvSet.size());
268      } else {
269        kvSet = allKVMap.values();
270        LOG.info("Wildcard scanner");
271        LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
272
273      }
274      InternalScanner scanner = region.getScanner(scan);
275      List<Cell> results = new ArrayList<>();
276      while (scanner.next(results))
277        ;
278      assertEquals(kvSet.size(), results.size());
279      assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
280    }
281
282    HBaseTestingUtil.closeRegionAndWAL(region);
283  }
284
285  List<String> generateRandomWords(int numberOfWords, String suffix) {
286    Set<String> wordSet = new HashSet<>();
287    for (int i = 0; i < numberOfWords; i++) {
288      int lengthOfWords = (int) (Math.random() * 5) + 1;
289      char[] wordChar = new char[lengthOfWords];
290      for (int j = 0; j < wordChar.length; j++) {
291        wordChar[j] = (char) (Math.random() * 26 + 97);
292      }
293      String word;
294      if (suffix == null) {
295        word = new String(wordChar);
296      } else {
297        word = new String(wordChar) + suffix;
298      }
299      wordSet.add(word);
300    }
301    List<String> wordList = new ArrayList<>(wordSet);
302    return wordList;
303  }
304
305}