001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertNull;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.nio.charset.StandardCharsets;
025import java.util.ArrayList;
026import java.util.List;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.Cell;
030import org.apache.hadoop.hbase.CellUtil;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtility;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.HRegionInfo;
035import org.apache.hadoop.hbase.HTableDescriptor;
036import org.apache.hadoop.hbase.MetaTableAccessor;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.Delete;
039import org.apache.hadoop.hbase.client.Durability;
040import org.apache.hadoop.hbase.client.Put;
041import org.apache.hadoop.hbase.client.RegionInfo;
042import org.apache.hadoop.hbase.client.RegionInfoBuilder;
043import org.apache.hadoop.hbase.client.Result;
044import org.apache.hadoop.hbase.client.Scan;
045import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
046import org.apache.hadoop.hbase.testclassification.RegionServerTests;
047import org.apache.hadoop.hbase.testclassification.SmallTests;
048import org.apache.hadoop.hbase.util.Bytes;
049import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
050import org.apache.hadoop.hbase.wal.WAL;
051import org.junit.ClassRule;
052import org.junit.Rule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.junit.rules.TestName;
056import org.slf4j.Logger;
057import org.slf4j.LoggerFactory;
058
059/**
060 * TestGet is a medley of tests of get all done up as a single test. It was originally written to
061 * test a method since removed, getClosestAtOrBefore but the test is retained because it runs some
062 * interesting exercises.
063 */
064@Category({ RegionServerTests.class, SmallTests.class })
065public class TestGetClosestAtOrBefore {
066
067  @ClassRule
068  public static final HBaseClassTestRule CLASS_RULE =
069    HBaseClassTestRule.forClass(TestGetClosestAtOrBefore.class);
070
071  @Rule
072  public TestName testName = new TestName();
073  private static final Logger LOG = LoggerFactory.getLogger(TestGetClosestAtOrBefore.class);
074
075  private static final byte[] T00 = Bytes.toBytes("000");
076  private static final byte[] T10 = Bytes.toBytes("010");
077  private static final byte[] T11 = Bytes.toBytes("011");
078  private static final byte[] T12 = Bytes.toBytes("012");
079  private static final byte[] T20 = Bytes.toBytes("020");
080  private static final byte[] T30 = Bytes.toBytes("030");
081  private static final byte[] T31 = Bytes.toBytes("031");
082  private static final byte[] T35 = Bytes.toBytes("035");
083  private static final byte[] T40 = Bytes.toBytes("040");
084
085  private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
086  private static Configuration conf = UTIL.getConfiguration();
087
088  @Test
089  public void testUsingMetaAndBinary() throws IOException {
090    Path rootdir = UTIL.getDataTestDirOnTestFS();
091    // Up flush size else we bind up when we use default catalog flush of 16k.
092    TableDescriptorBuilder metaBuilder =
093      UTIL.getMetaTableDescriptorBuilder().setMemStoreFlushSize(64 * 1024 * 1024);
094    HRegion mr = HBaseTestingUtility.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO, rootdir,
095      this.conf, metaBuilder.build());
096    try {
097      // Write rows for three tables 'A', 'B', and 'C'.
098      for (char c = 'A'; c < 'D'; c++) {
099        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("" + c));
100        final int last = 128;
101        final int interval = 2;
102        for (int i = 0; i <= last; i += interval) {
103          RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName())
104            .setStartKey(i == 0 ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i))
105            .setEndKey(i == last ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i + interval))
106            .build();
107          Put put =
108            MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
109          put.setDurability(Durability.SKIP_WAL);
110          LOG.info("Put {}", put);
111          mr.put(put);
112        }
113      }
114      InternalScanner s = mr.getScanner(new Scan());
115      try {
116        List<Cell> keys = new ArrayList<>();
117        while (s.next(keys)) {
118          LOG.info("Scan {}", keys);
119          keys.clear();
120        }
121      } finally {
122        s.close();
123      }
124      findRow(mr, 'C', 44, 44);
125      findRow(mr, 'C', 45, 44);
126      findRow(mr, 'C', 46, 46);
127      findRow(mr, 'C', 43, 42);
128      mr.flush(true);
129      findRow(mr, 'C', 44, 44);
130      findRow(mr, 'C', 45, 44);
131      findRow(mr, 'C', 46, 46);
132      findRow(mr, 'C', 43, 42);
133      // Now delete 'C' and make sure I don't get entries from 'B'.
134      byte[] firstRowInC = RegionInfo.createRegionName(TableName.valueOf("" + 'C'),
135        HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES, false);
136      Scan scan = new Scan().withStartRow(firstRowInC);
137      s = mr.getScanner(scan);
138      try {
139        List<Cell> keys = new ArrayList<>();
140        while (s.next(keys)) {
141          LOG.info("Delete {}", keys);
142          mr.delete(new Delete(CellUtil.cloneRow(keys.get(0))));
143          keys.clear();
144        }
145      } finally {
146        s.close();
147      }
148      // Assert we get null back (pass -1).
149      findRow(mr, 'C', 44, -1);
150      findRow(mr, 'C', 45, -1);
151      findRow(mr, 'C', 46, -1);
152      findRow(mr, 'C', 43, -1);
153      mr.flush(true);
154      findRow(mr, 'C', 44, -1);
155      findRow(mr, 'C', 45, -1);
156      findRow(mr, 'C', 46, -1);
157      findRow(mr, 'C', 43, -1);
158    } finally {
159      HBaseTestingUtility.closeRegionAndWAL(mr);
160    }
161  }
162
163  /*
164   * @param answer Pass -1 if we're not to find anything.
165   * @return Row found.
166   */
167  private byte[] findRow(final Region mr, final char table, final int rowToFind, final int answer)
168    throws IOException {
169    TableName tableb = TableName.valueOf("" + table);
170    // Find the row.
171    byte[] tofindBytes = Bytes.toBytes((short) rowToFind);
172    byte[] metaKey = HRegionInfo.createRegionName(tableb, tofindBytes, HConstants.NINES, false);
173    LOG.info("find=" + new String(metaKey, StandardCharsets.UTF_8));
174    Result r = UTIL.getClosestRowBefore(mr, metaKey, HConstants.CATALOG_FAMILY);
175    if (answer == -1) {
176      assertNull(r);
177      return null;
178    }
179    assertTrue(
180      Bytes.compareTo(Bytes.toBytes((short) answer), extractRowFromMetaRow(r.getRow())) == 0);
181    return r.getRow();
182  }
183
184  private byte[] extractRowFromMetaRow(final byte[] b) {
185    int firstDelimiter = Bytes.searchDelimiterIndex(b, 0, b.length, HConstants.DELIMITER);
186    int lastDelimiter = Bytes.searchDelimiterIndexInReverse(b, 0, b.length, HConstants.DELIMITER);
187    int length = lastDelimiter - firstDelimiter - 1;
188    byte[] row = new byte[length];
189    System.arraycopy(b, firstDelimiter + 1, row, 0, length);
190    return row;
191  }
192
193  /**
194   * Test file of multiple deletes and with deletes as final key.
195   * @see <a href="https://issues.apache.org/jira/browse/HBASE-751">HBASE-751</a>
196   */
197  @Test
198  public void testGetClosestRowBefore3() throws IOException {
199    HRegion region = null;
200    byte[] c0 = UTIL.COLUMNS[0];
201    byte[] c1 = UTIL.COLUMNS[1];
202    try {
203      TableName tn = TableName.valueOf(testName.getMethodName());
204      HTableDescriptor htd = UTIL.createTableDescriptor(tn);
205      region = UTIL.createLocalHRegion(htd, null, null);
206
207      Put p = new Put(T00);
208      p.addColumn(c0, c0, T00);
209      region.put(p);
210
211      p = new Put(T10);
212      p.addColumn(c0, c0, T10);
213      region.put(p);
214
215      p = new Put(T20);
216      p.addColumn(c0, c0, T20);
217      region.put(p);
218
219      Result r = UTIL.getClosestRowBefore(region, T20, c0);
220      assertTrue(Bytes.equals(T20, r.getRow()));
221
222      Delete d = new Delete(T20);
223      d.addColumn(c0, c0);
224      region.delete(d);
225
226      r = UTIL.getClosestRowBefore(region, T20, c0);
227      assertTrue(Bytes.equals(T10, r.getRow()));
228
229      p = new Put(T30);
230      p.addColumn(c0, c0, T30);
231      region.put(p);
232
233      r = UTIL.getClosestRowBefore(region, T30, c0);
234      assertTrue(Bytes.equals(T30, r.getRow()));
235
236      d = new Delete(T30);
237      d.addColumn(c0, c0);
238      region.delete(d);
239
240      r = UTIL.getClosestRowBefore(region, T30, c0);
241      assertTrue(Bytes.equals(T10, r.getRow()));
242      r = UTIL.getClosestRowBefore(region, T31, c0);
243      assertTrue(Bytes.equals(T10, r.getRow()));
244
245      region.flush(true);
246
247      // try finding "010" after flush
248      r = UTIL.getClosestRowBefore(region, T30, c0);
249      assertTrue(Bytes.equals(T10, r.getRow()));
250      r = UTIL.getClosestRowBefore(region, T31, c0);
251      assertTrue(Bytes.equals(T10, r.getRow()));
252
253      // Put into a different column family. Should make it so I still get t10
254      p = new Put(T20);
255      p.addColumn(c1, c1, T20);
256      region.put(p);
257
258      r = UTIL.getClosestRowBefore(region, T30, c0);
259      assertTrue(Bytes.equals(T10, r.getRow()));
260      r = UTIL.getClosestRowBefore(region, T31, c0);
261      assertTrue(Bytes.equals(T10, r.getRow()));
262
263      region.flush(true);
264
265      r = UTIL.getClosestRowBefore(region, T30, c0);
266      assertTrue(Bytes.equals(T10, r.getRow()));
267      r = UTIL.getClosestRowBefore(region, T31, c0);
268      assertTrue(Bytes.equals(T10, r.getRow()));
269
270      // Now try combo of memcache and mapfiles. Delete the t20 COLUMS[1]
271      // in memory; make sure we get back t10 again.
272      d = new Delete(T20);
273      d.addColumn(c1, c1);
274      region.delete(d);
275      r = UTIL.getClosestRowBefore(region, T30, c0);
276      assertTrue(Bytes.equals(T10, r.getRow()));
277
278      // Ask for a value off the end of the file. Should return t10.
279      r = UTIL.getClosestRowBefore(region, T31, c0);
280      assertTrue(Bytes.equals(T10, r.getRow()));
281      region.flush(true);
282      r = UTIL.getClosestRowBefore(region, T31, c0);
283      assertTrue(Bytes.equals(T10, r.getRow()));
284
285      // Ok. Let the candidate come out of hfile but have delete of
286      // the candidate be in memory.
287      p = new Put(T11);
288      p.addColumn(c0, c0, T11);
289      region.put(p);
290      d = new Delete(T10);
291      d.addColumn(c1, c1);
292      r = UTIL.getClosestRowBefore(region, T12, c0);
293      assertTrue(Bytes.equals(T11, r.getRow()));
294    } finally {
295      if (region != null) {
296        try {
297          WAL wal = region.getWAL();
298          region.close();
299          wal.close();
300        } catch (Exception e) {
301          e.printStackTrace();
302        }
303      }
304    }
305  }
306
307  /** For HBASE-694 */
308  @Test
309  public void testGetClosestRowBefore2() throws IOException {
310    HRegion region = null;
311    byte[] c0 = UTIL.COLUMNS[0];
312    try {
313      TableName tn = TableName.valueOf(testName.getMethodName());
314      HTableDescriptor htd = UTIL.createTableDescriptor(tn);
315      region = UTIL.createLocalHRegion(htd, null, null);
316
317      Put p = new Put(T10);
318      p.addColumn(c0, c0, T10);
319      region.put(p);
320
321      p = new Put(T30);
322      p.addColumn(c0, c0, T30);
323      region.put(p);
324
325      p = new Put(T40);
326      p.addColumn(c0, c0, T40);
327      region.put(p);
328
329      // try finding "035"
330      Result r = UTIL.getClosestRowBefore(region, T35, c0);
331      assertTrue(Bytes.equals(T30, r.getRow()));
332
333      region.flush(true);
334
335      // try finding "035"
336      r = UTIL.getClosestRowBefore(region, T35, c0);
337      assertTrue(Bytes.equals(T30, r.getRow()));
338
339      p = new Put(T20);
340      p.addColumn(c0, c0, T20);
341      region.put(p);
342
343      // try finding "035"
344      r = UTIL.getClosestRowBefore(region, T35, c0);
345      assertTrue(Bytes.equals(T30, r.getRow()));
346
347      region.flush(true);
348
349      // try finding "035"
350      r = UTIL.getClosestRowBefore(region, T35, c0);
351      assertTrue(Bytes.equals(T30, r.getRow()));
352    } finally {
353      if (region != null) {
354        try {
355          WAL wal = region.getWAL();
356          region.close();
357          wal.close();
358        } catch (Exception e) {
359          e.printStackTrace();
360        }
361      }
362    }
363  }
364
365}