001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.CellUtil.createCell;
021import static org.apache.hadoop.hbase.KeyValueTestUtil.create;
022import static org.apache.hadoop.hbase.regionserver.KeyValueScanFixture.scanFixture;
023import static org.junit.Assert.assertEquals;
024import static org.junit.Assert.assertFalse;
025import static org.junit.Assert.assertNull;
026import static org.junit.Assert.assertSame;
027import static org.junit.Assert.assertTrue;
028
029import java.io.IOException;
030import java.util.ArrayList;
031import java.util.Arrays;
032import java.util.Collections;
033import java.util.List;
034import java.util.NavigableSet;
035import java.util.TreeSet;
036import java.util.concurrent.atomic.AtomicInteger;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.hbase.Cell;
039import org.apache.hadoop.hbase.CellComparator;
040import org.apache.hadoop.hbase.CellUtil;
041import org.apache.hadoop.hbase.CompareOperator;
042import org.apache.hadoop.hbase.HBaseClassTestRule;
043import org.apache.hadoop.hbase.HBaseConfiguration;
044import org.apache.hadoop.hbase.HBaseTestingUtility;
045import org.apache.hadoop.hbase.HConstants;
046import org.apache.hadoop.hbase.KeepDeletedCells;
047import org.apache.hadoop.hbase.KeyValue;
048import org.apache.hadoop.hbase.PrivateCellUtil;
049import org.apache.hadoop.hbase.client.Get;
050import org.apache.hadoop.hbase.client.Scan;
051import org.apache.hadoop.hbase.filter.BinaryComparator;
052import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
053import org.apache.hadoop.hbase.filter.Filter;
054import org.apache.hadoop.hbase.filter.QualifierFilter;
055import org.apache.hadoop.hbase.testclassification.RegionServerTests;
056import org.apache.hadoop.hbase.testclassification.SmallTests;
057import org.apache.hadoop.hbase.util.Bytes;
058import org.apache.hadoop.hbase.util.CollectionBackedScanner;
059import org.apache.hadoop.hbase.util.EnvironmentEdge;
060import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
061import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
062import org.junit.ClassRule;
063import org.junit.Ignore;
064import org.junit.Rule;
065import org.junit.Test;
066import org.junit.experimental.categories.Category;
067import org.junit.rules.TestName;
068import org.slf4j.Logger;
069import org.slf4j.LoggerFactory;
070
071// Can't be small as it plays with EnvironmentEdgeManager
072@Category({ RegionServerTests.class, SmallTests.class })
073public class TestStoreScanner {
074
075  @ClassRule
076  public static final HBaseClassTestRule CLASS_RULE =
077    HBaseClassTestRule.forClass(TestStoreScanner.class);
078
079  private static final Logger LOG = LoggerFactory.getLogger(TestStoreScanner.class);
080  @Rule
081  public TestName name = new TestName();
082  private static final String CF_STR = "cf";
083  private static final byte[] CF = Bytes.toBytes(CF_STR);
084  static Configuration CONF = HBaseConfiguration.create();
085  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
086  private ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, Integer.MAX_VALUE, Long.MAX_VALUE,
087    KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
088
089  /**
090   * From here on down, we have a bunch of defines and specific CELL_GRID of Cells. The CELL_GRID
091   * then has a Scanner that can fake out 'block' transitions. All this elaborate setup is for tests
092   * that ensure we don't overread, and that the {@link StoreScanner} is not overly enthusiastic.
093   */
094  private static final byte[] ZERO = new byte[] { '0' };
095  private static final byte[] ZERO_POINT_ZERO = new byte[] { '0', '.', '0' };
096  private static final byte[] ONE = new byte[] { '1' };
097  private static final byte[] TWO = new byte[] { '2' };
098  private static final byte[] TWO_POINT_TWO = new byte[] { '2', '.', '2' };
099  private static final byte[] THREE = new byte[] { '3' };
100  private static final byte[] FOUR = new byte[] { '4' };
101  private static final byte[] FIVE = new byte[] { '5' };
102  private static final byte[] VALUE = new byte[] { 'v' };
103  private static final int CELL_GRID_BLOCK2_BOUNDARY = 4;
104  private static final int CELL_GRID_BLOCK3_BOUNDARY = 11;
105  private static final int CELL_GRID_BLOCK4_BOUNDARY = 15;
106  private static final int CELL_GRID_BLOCK5_BOUNDARY = 19;
107
108  /**
109   * Five rows by four columns distinguished by column qualifier (column qualifier is one of the
110   * four rows... ONE, TWO, etc.). Exceptions are a weird row after TWO; it is TWO_POINT_TWO. And
111   * then row FOUR has five columns finishing w/ row FIVE having a single column. We will use this
112   * to test scan does the right thing as it we do Gets, StoreScanner#optimize, and what we do on
113   * (faked) block boundaries.
114   */
115  private static final Cell[] CELL_GRID =
116    new Cell[] { createCell(ONE, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE),
117      createCell(ONE, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE),
118      createCell(ONE, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE),
119      createCell(ONE, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE),
120      // Offset 4 CELL_GRID_BLOCK2_BOUNDARY
121      createCell(TWO, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE),
122      createCell(TWO, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE),
123      createCell(TWO, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE),
124      createCell(TWO, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE),
125      createCell(TWO_POINT_TWO, CF, ZERO, 1L, KeyValue.Type.Put.getCode(), VALUE),
126      createCell(TWO_POINT_TWO, CF, ZERO_POINT_ZERO, 1L, KeyValue.Type.Put.getCode(), VALUE),
127      createCell(TWO_POINT_TWO, CF, FIVE, 1L, KeyValue.Type.Put.getCode(), VALUE),
128      // Offset 11! CELL_GRID_BLOCK3_BOUNDARY
129      createCell(THREE, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE),
130      createCell(THREE, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE),
131      createCell(THREE, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE),
132      createCell(THREE, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE),
133      // Offset 15 CELL_GRID_BLOCK4_BOUNDARY
134      createCell(FOUR, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE),
135      createCell(FOUR, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE),
136      createCell(FOUR, CF, THREE, 1L, KeyValue.Type.Put.getCode(), VALUE),
137      createCell(FOUR, CF, FOUR, 1L, KeyValue.Type.Put.getCode(), VALUE),
138      // Offset 19 CELL_GRID_BLOCK5_BOUNDARY
139      createCell(FOUR, CF, FIVE, 1L, KeyValue.Type.Put.getCode(), VALUE),
140      createCell(FIVE, CF, ZERO, 1L, KeyValue.Type.Put.getCode(), VALUE), };
141
142  private static class KeyValueHeapWithCount extends KeyValueHeap {
143
144    final AtomicInteger count;
145
146    public KeyValueHeapWithCount(List<? extends KeyValueScanner> scanners,
147      CellComparator comparator, AtomicInteger count) throws IOException {
148      super(scanners, comparator);
149      this.count = count;
150    }
151
152    @Override
153    public Cell peek() {
154      this.count.incrementAndGet();
155      return super.peek();
156    }
157  }
158
159  /**
160   * A StoreScanner for our CELL_GRID above. Fakes the block transitions. Does counts of calls to
161   * optimize and counts of when optimize actually did an optimize.
162   */
163  private static class CellGridStoreScanner extends StoreScanner {
164    // Count of how often optimize is called and of how often it does an optimize.
165    AtomicInteger count;
166    final AtomicInteger optimization = new AtomicInteger(0);
167
168    CellGridStoreScanner(final Scan scan, ScanInfo scanInfo) throws IOException {
169      super(scan, scanInfo, scan.getFamilyMap().get(CF),
170        Arrays.<KeyValueScanner> asList(new KeyValueScanner[] {
171          new KeyValueScanFixture(CellComparator.getInstance(), CELL_GRID) }));
172    }
173
174    @Override
175    protected void resetKVHeap(List<? extends KeyValueScanner> scanners, CellComparator comparator)
176      throws IOException {
177      if (count == null) {
178        count = new AtomicInteger(0);
179      }
180      heap = newKVHeap(scanners, comparator);
181    }
182
183    @Override
184    protected KeyValueHeap newKVHeap(List<? extends KeyValueScanner> scanners,
185      CellComparator comparator) throws IOException {
186      return new KeyValueHeapWithCount(scanners, comparator, count);
187    }
188
189    @Override
190    protected boolean trySkipToNextRow(Cell cell) throws IOException {
191      boolean optimized = super.trySkipToNextRow(cell);
192      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
193        + ", optimized=" + optimized);
194      if (optimized) {
195        optimization.incrementAndGet();
196      }
197      return optimized;
198    }
199
200    @Override
201    protected boolean trySkipToNextColumn(Cell cell) throws IOException {
202      boolean optimized = super.trySkipToNextColumn(cell);
203      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
204        + ", optimized=" + optimized);
205      if (optimized) {
206        optimization.incrementAndGet();
207      }
208      return optimized;
209    }
210
211    @Override
212    public Cell getNextIndexedKey() {
213      // Fake block boundaries by having index of next block change as we go through scan.
214      return count.get() > CELL_GRID_BLOCK4_BOUNDARY
215        ? PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK5_BOUNDARY])
216        : count.get() > CELL_GRID_BLOCK3_BOUNDARY
217          ? PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK4_BOUNDARY])
218        : count.get() > CELL_GRID_BLOCK2_BOUNDARY
219          ? PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK3_BOUNDARY])
220        : PrivateCellUtil.createFirstOnRow(CELL_GRID[CELL_GRID_BLOCK2_BOUNDARY]);
221    }
222  };
223
224  private static final int CELL_WITH_VERSIONS_BLOCK2_BOUNDARY = 4;
225
226  private static final Cell[] CELL_WITH_VERSIONS =
227    new Cell[] { createCell(ONE, CF, ONE, 2L, KeyValue.Type.Put.getCode(), VALUE),
228      createCell(ONE, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE),
229      createCell(ONE, CF, TWO, 2L, KeyValue.Type.Put.getCode(), VALUE),
230      createCell(ONE, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE),
231      // Offset 4 CELL_WITH_VERSIONS_BLOCK2_BOUNDARY
232      createCell(TWO, CF, ONE, 1L, KeyValue.Type.Put.getCode(), VALUE),
233      createCell(TWO, CF, TWO, 1L, KeyValue.Type.Put.getCode(), VALUE), };
234
235  private static class CellWithVersionsStoreScanner extends StoreScanner {
236    // Count of how often optimize is called and of how often it does an optimize.
237    final AtomicInteger optimization = new AtomicInteger(0);
238
239    CellWithVersionsStoreScanner(final Scan scan, ScanInfo scanInfo) throws IOException {
240      super(scan, scanInfo, scan.getFamilyMap().get(CF),
241        Arrays.<KeyValueScanner> asList(new KeyValueScanner[] {
242          new KeyValueScanFixture(CellComparator.getInstance(), CELL_WITH_VERSIONS) }));
243    }
244
245    @Override
246    protected boolean trySkipToNextColumn(Cell cell) throws IOException {
247      boolean optimized = super.trySkipToNextColumn(cell);
248      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
249        + ", optimized=" + optimized);
250      if (optimized) {
251        optimization.incrementAndGet();
252      }
253      return optimized;
254    }
255
256    @Override
257    public Cell getNextIndexedKey() {
258      // Fake block boundaries by having index of next block change as we go through scan.
259      return PrivateCellUtil
260        .createFirstOnRow(CELL_WITH_VERSIONS[CELL_WITH_VERSIONS_BLOCK2_BOUNDARY]);
261    }
262  };
263
264  private static class CellWithVersionsNoOptimizeStoreScanner extends StoreScanner {
265    // Count of how often optimize is called and of how often it does an optimize.
266    final AtomicInteger optimization = new AtomicInteger(0);
267
268    CellWithVersionsNoOptimizeStoreScanner(Scan scan, ScanInfo scanInfo) throws IOException {
269      super(scan, scanInfo, scan.getFamilyMap().get(CF),
270        Arrays.<KeyValueScanner> asList(new KeyValueScanner[] {
271          new KeyValueScanFixture(CellComparator.getInstance(), CELL_WITH_VERSIONS) }));
272    }
273
274    @Override
275    protected boolean trySkipToNextColumn(Cell cell) throws IOException {
276      boolean optimized = super.trySkipToNextColumn(cell);
277      LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
278        + ", optimized=" + optimized);
279      if (optimized) {
280        optimization.incrementAndGet();
281      }
282      return optimized;
283    }
284
285    @Override
286    public Cell getNextIndexedKey() {
287      return null;
288    }
289  };
290
291  @Test
292  public void testWithColumnCountGetFilter() throws Exception {
293    Get get = new Get(ONE);
294    get.readAllVersions();
295    get.addFamily(CF);
296    get.setFilter(new ColumnCountGetFilter(2));
297
298    try (CellWithVersionsNoOptimizeStoreScanner scannerNoOptimize =
299      new CellWithVersionsNoOptimizeStoreScanner(new Scan(get), this.scanInfo)) {
300      List<Cell> results = new ArrayList<>();
301      while (scannerNoOptimize.next(results)) {
302        continue;
303      }
304      assertEquals(2, results.size());
305      assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0]));
306      assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2]));
307      assertTrue("Optimize should do some optimizations",
308        scannerNoOptimize.optimization.get() == 0);
309    }
310
311    get.setFilter(new ColumnCountGetFilter(2));
312    try (CellWithVersionsStoreScanner scanner =
313      new CellWithVersionsStoreScanner(new Scan(get), this.scanInfo)) {
314      List<Cell> results = new ArrayList<>();
315      while (scanner.next(results)) {
316        continue;
317      }
318      assertEquals(2, results.size());
319      assertTrue(CellUtil.matchingColumn(results.get(0), CELL_WITH_VERSIONS[0]));
320      assertTrue(CellUtil.matchingColumn(results.get(1), CELL_WITH_VERSIONS[2]));
321      assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0);
322    }
323  }
324
325  /*
326   * Test utility for building a NavigableSet for scanners.
327   */
328  NavigableSet<byte[]> getCols(String... strCols) {
329    NavigableSet<byte[]> cols = new TreeSet<>(Bytes.BYTES_COMPARATOR);
330    for (String col : strCols) {
331      byte[] bytes = Bytes.toBytes(col);
332      cols.add(bytes);
333    }
334    return cols;
335  }
336
337  @Test
338  public void testFullRowGetDoesNotOverreadWhenRowInsideOneBlock() throws IOException {
339    // Do a Get against row two. Row two is inside a block that starts with row TWO but ends with
340    // row TWO_POINT_TWO. We should read one block only.
341    Get get = new Get(TWO);
342    Scan scan = new Scan(get);
343    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
344      List<Cell> results = new ArrayList<>();
345      while (scanner.next(results)) {
346        continue;
347      }
348      // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the
349      // TWO_POINT_TWO row does not have a a column ONE.
350      assertEquals(4, results.size());
351      // We should have gone the optimize route 5 times totally... an INCLUDE for the four cells
352      // in the row plus the DONE on the end.
353      assertEquals(5, scanner.count.get());
354      assertEquals(1, scanner.memstoreOnlyReads);
355      // For a full row Get, there should be no opportunity for scanner optimization.
356      assertEquals(0, scanner.optimization.get());
357    }
358  }
359
360  @Test
361  public void testFullRowSpansBlocks() throws IOException {
362    // Do a Get against row FOUR. It spans two blocks.
363    Get get = new Get(FOUR);
364    Scan scan = new Scan(get);
365    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
366      List<Cell> results = new ArrayList<>();
367      while (scanner.next(results)) {
368        continue;
369      }
370      // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the
371      // TWO_POINT_TWO row does not have a a column ONE.
372      assertEquals(5, results.size());
373      // We should have gone the optimize route 6 times totally... an INCLUDE for the five cells
374      // in the row plus the DONE on the end.
375      assertEquals(6, scanner.count.get());
376      // For a full row Get, there should be no opportunity for scanner optimization.
377      assertEquals(0, scanner.optimization.get());
378    }
379  }
380
381  /**
382   * Test optimize in StoreScanner. Test that we skip to the next 'block' when we it makes sense
383   * reading the block 'index'.
384   */
385  @Test
386  public void testOptimize() throws IOException {
387    Scan scan = new Scan();
388    // A scan that just gets the first qualifier on each row of the CELL_GRID
389    scan.addColumn(CF, ONE);
390    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
391      List<Cell> results = new ArrayList<>();
392      while (scanner.next(results)) {
393        continue;
394      }
395      // Should be four results of column 1 (though there are 5 rows in the CELL_GRID -- the
396      // TWO_POINT_TWO row does not have a a column ONE.
397      assertEquals(4, results.size());
398      for (Cell cell : results) {
399        assertTrue(Bytes.equals(ONE, 0, ONE.length, cell.getQualifierArray(),
400          cell.getQualifierOffset(), cell.getQualifierLength()));
401      }
402      assertTrue("Optimize should do some optimizations", scanner.optimization.get() > 0);
403    }
404  }
405
406  /**
407   * Ensure the optimize Scan method in StoreScanner does not get in the way of a Get doing minimum
408   * work... seeking to start of block and then SKIPPING until we find the wanted Cell. This
409   * 'simple' scenario mimics case of all Cells fitting inside a single HFileBlock. See HBASE-15392.
410   * This test is a little cryptic. Takes a bit of staring to figure what it up to.
411   */
412  @Test
413  public void testOptimizeAndGet() throws IOException {
414    // First test a Get of two columns in the row R2. Every Get is a Scan. Get columns named
415    // R2 and R3.
416    Get get = new Get(TWO);
417    get.addColumn(CF, TWO);
418    get.addColumn(CF, THREE);
419    Scan scan = new Scan(get);
420    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
421      List<Cell> results = new ArrayList<>();
422      // For a Get there should be no more next's after the first call.
423      assertEquals(false, scanner.next(results));
424      // Should be one result only.
425      assertEquals(2, results.size());
426      // And we should have gone through optimize twice only.
427      assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", 3,
428        scanner.count.get());
429      assertEquals("Memstore Read count should be", 1, scanner.memstoreOnlyReads);
430    }
431  }
432
433  /**
434   * Ensure that optimize does not cause the Get to do more seeking than required. Optimize (see
435   * HBASE-15392) was causing us to seek all Cells in a block when a Get Scan if the next block
436   * index/start key was a different row to the current one. A bug. We'd call next too often because
437   * we had to exhaust all Cells in the current row making us load the next block just to discard
438   * what we read there. This test is a little cryptic. Takes a bit of staring to figure what it up
439   * to.
440   */
441  @Test
442  public void testOptimizeAndGetWithFakedNextBlockIndexStart() throws IOException {
443    // First test a Get of second column in the row R2. Every Get is a Scan. Second column has a
444    // qualifier of R2.
445    Get get = new Get(THREE);
446    get.addColumn(CF, TWO);
447    Scan scan = new Scan(get);
448    try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
449      List<Cell> results = new ArrayList<>();
450      // For a Get there should be no more next's after the first call.
451      assertEquals(false, scanner.next(results));
452      // Should be one result only.
453      assertEquals(1, results.size());
454      // And we should have gone through optimize twice only.
455      assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", 2,
456        scanner.count.get());
457    }
458  }
459
460  @Test
461  public void testScanTimeRange() throws IOException {
462    String r1 = "R1";
463    // returns only 1 of these 2 even though same timestamp
464    KeyValue[] kvs = new KeyValue[] { create(r1, CF_STR, "a", 1, KeyValue.Type.Put, "dont-care"),
465      create(r1, CF_STR, "a", 2, KeyValue.Type.Put, "dont-care"),
466      create(r1, CF_STR, "a", 3, KeyValue.Type.Put, "dont-care"),
467      create(r1, CF_STR, "a", 4, KeyValue.Type.Put, "dont-care"),
468      create(r1, CF_STR, "a", 5, KeyValue.Type.Put, "dont-care"), };
469    List<KeyValueScanner> scanners = Arrays.<KeyValueScanner> asList(
470      new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), kvs) });
471    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
472    scanSpec.setTimeRange(0, 6);
473    scanSpec.readAllVersions();
474    List<Cell> results = null;
475    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
476      results = new ArrayList<>();
477      assertEquals(true, scan.next(results));
478      assertEquals(5, results.size());
479      assertEquals(kvs[kvs.length - 1], results.get(0));
480    }
481    // Scan limited TimeRange
482    scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
483    scanSpec.setTimeRange(1, 3);
484    scanSpec.readAllVersions();
485    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
486      results = new ArrayList<>();
487      assertEquals(true, scan.next(results));
488      assertEquals(2, results.size());
489    }
490    // Another range.
491    scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
492    scanSpec.setTimeRange(5, 10);
493    scanSpec.readAllVersions();
494    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
495      results = new ArrayList<>();
496      assertEquals(true, scan.next(results));
497      assertEquals(1, results.size());
498    }
499    // See how TimeRange and Versions interact.
500    // Another range.
501    scanSpec = new Scan().withStartRow(Bytes.toBytes(r1));
502    scanSpec.setTimeRange(0, 10);
503    scanSpec.readVersions(3);
504    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
505      results = new ArrayList<>();
506      assertEquals(true, scan.next(results));
507      assertEquals(3, results.size());
508    }
509  }
510
511  @Test
512  public void testScanSameTimestamp() throws IOException {
513    // returns only 1 of these 2 even though same timestamp
514    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
515      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), };
516    List<KeyValueScanner> scanners = Arrays
517      .asList(new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), kvs) });
518
519    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
520    // this only uses maxVersions (default=1) and TimeRange (default=all)
521    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
522      List<Cell> results = new ArrayList<>();
523      assertEquals(true, scan.next(results));
524      assertEquals(1, results.size());
525      assertEquals(1, scan.memstoreOnlyReads);
526      assertEquals(kvs[0], results.get(0));
527    }
528  }
529
530  @Test
531  public void testNonUserScan() throws IOException {
532    // returns only 1 of these 2 even though same timestamp
533    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
534      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), };
535    List<KeyValueScanner> scanners = Arrays
536      .asList(new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), kvs) });
537
538    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
539    // this only uses maxVersions (default=1) and TimeRange (default=all)
540    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners,
541      ScanType.COMPACT_RETAIN_DELETES)) {
542      List<Cell> results = new ArrayList<>();
543      assertEquals(true, scan.next(results));
544      assertEquals(1, results.size());
545      assertEquals(0, scan.memstoreOnlyReads);
546      assertEquals(kvs[0], results.get(0));
547    }
548  }
549
550  /*
551   * Test test shows exactly how the matcher's return codes confuses the StoreScanner and prevent it
552   * from doing the right thing. Seeking once, then nexting twice should return R1, then R2, but in
553   * this case it doesnt. TODO this comment makes no sense above. Appears to do the right thing.
554   */
555  @Test
556  public void testWontNextToNext() throws IOException {
557    // build the scan file:
558    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
559      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
560      create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care") };
561    List<KeyValueScanner> scanners = scanFixture(kvs);
562
563    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
564    // this only uses maxVersions (default=1) and TimeRange (default=all)
565    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
566      List<Cell> results = new ArrayList<>();
567      scan.next(results);
568      assertEquals(1, results.size());
569      assertEquals(kvs[0], results.get(0));
570      // should be ok...
571      // now scan _next_ again.
572      results.clear();
573      scan.next(results);
574      assertEquals(1, results.size());
575      assertEquals(kvs[2], results.get(0));
576
577      results.clear();
578      scan.next(results);
579      assertEquals(0, results.size());
580    }
581  }
582
583  @Test
584  public void testDeleteVersionSameTimestamp() throws IOException {
585    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
586      create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), };
587    List<KeyValueScanner> scanners = scanFixture(kvs);
588    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
589    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
590      List<Cell> results = new ArrayList<>();
591      assertFalse(scan.next(results));
592      assertEquals(0, results.size());
593    }
594  }
595
596  /*
597   * Test the case where there is a delete row 'in front of' the next row, the scanner will move to
598   * the next row.
599   */
600  @Test
601  public void testDeletedRowThenGoodRow() throws IOException {
602    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
603      create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
604      create("R2", "cf", "a", 20, KeyValue.Type.Put, "dont-care") };
605    List<KeyValueScanner> scanners = scanFixture(kvs);
606    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
607    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
608      List<Cell> results = new ArrayList<>();
609      assertEquals(true, scan.next(results));
610      assertEquals(0, results.size());
611
612      assertEquals(true, scan.next(results));
613      assertEquals(1, results.size());
614      assertEquals(kvs[2], results.get(0));
615
616      assertEquals(false, scan.next(results));
617    }
618  }
619
620  @Test
621  public void testDeleteVersionMaskingMultiplePuts() throws IOException {
622    long now = EnvironmentEdgeManager.currentTime();
623    KeyValue[] kvs1 = new KeyValue[] { create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
624      create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") };
625    KeyValue[] kvs2 =
626      new KeyValue[] { create("R1", "cf", "a", now - 500, KeyValue.Type.Put, "dont-care"),
627        create("R1", "cf", "a", now - 100, KeyValue.Type.Put, "dont-care"),
628        create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care") };
629    List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2);
630
631    try (StoreScanner scan = new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")),
632      scanInfo, getCols("a"), scanners)) {
633      List<Cell> results = new ArrayList<>();
634      // the two put at ts=now will be masked by the 1 delete, and
635      // since the scan default returns 1 version we'll return the newest
636      // key, which is kvs[2], now-100.
637      assertEquals(true, scan.next(results));
638      assertEquals(1, results.size());
639      assertEquals(kvs2[1], results.get(0));
640    }
641  }
642
643  @Test
644  public void testDeleteVersionsMixedAndMultipleVersionReturn() throws IOException {
645    long now = EnvironmentEdgeManager.currentTime();
646    KeyValue[] kvs1 = new KeyValue[] { create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
647      create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") };
648    KeyValue[] kvs2 =
649      new KeyValue[] { create("R1", "cf", "a", now - 500, KeyValue.Type.Put, "dont-care"),
650        create("R1", "cf", "a", now + 500, KeyValue.Type.Put, "dont-care"),
651        create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
652        create("R2", "cf", "z", now, KeyValue.Type.Put, "dont-care") };
653    List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2);
654
655    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2);
656    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
657      List<Cell> results = new ArrayList<>();
658      assertEquals(true, scan.next(results));
659      assertEquals(2, results.size());
660      assertEquals(kvs2[1], results.get(0));
661      assertEquals(kvs2[0], results.get(1));
662    }
663  }
664
665  @Test
666  public void testWildCardOneVersionScan() throws IOException {
667    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
668      create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"),
669      create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), };
670    List<KeyValueScanner> scanners = scanFixture(kvs);
671    try (StoreScanner scan =
672      new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")), scanInfo, null, scanners)) {
673      List<Cell> results = new ArrayList<>();
674      assertEquals(true, scan.next(results));
675      assertEquals(2, results.size());
676      assertEquals(kvs[0], results.get(0));
677      assertEquals(kvs[1], results.get(1));
678    }
679  }
680
681  @Test
682  public void testWildCardScannerUnderDeletes() throws IOException {
683    KeyValue[] kvs = new KeyValue[] {
684      // inc
685      create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
686      // orphaned delete column.
687      create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"),
688      // column b
689      // inc
690      create("R1", "cf", "b", 2, KeyValue.Type.Put, "dont-care"),
691      // inc
692      create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"),
693      // column c
694      create("R1", "cf", "c", 10, KeyValue.Type.Delete, "dont-care"),
695      // no
696      create("R1", "cf", "c", 10, KeyValue.Type.Put, "dont-care"),
697      // inc
698      create("R1", "cf", "c", 9, KeyValue.Type.Put, "dont-care"),
699      // column d
700      // inc
701      create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
702      create("R1", "cf", "d", 10, KeyValue.Type.DeleteColumn, "dont-care"),
703      // no
704      create("R1", "cf", "d", 9, KeyValue.Type.Put, "dont-care"),
705      // no
706      create("R1", "cf", "d", 8, KeyValue.Type.Put, "dont-care"),
707
708    };
709    List<KeyValueScanner> scanners = scanFixture(kvs);
710    try (
711      StoreScanner scan = new StoreScanner(new Scan().readVersions(2), scanInfo, null, scanners)) {
712      List<Cell> results = new ArrayList<>();
713      assertEquals(true, scan.next(results));
714      assertEquals(5, results.size());
715      assertEquals(kvs[0], results.get(0));
716      assertEquals(kvs[2], results.get(1));
717      assertEquals(kvs[3], results.get(2));
718      assertEquals(kvs[6], results.get(3));
719      assertEquals(kvs[7], results.get(4));
720    }
721  }
722
723  @Test
724  public void testDeleteFamily() throws IOException {
725    KeyValue[] kvs =
726      new KeyValue[] { create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"),
727        create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
728        create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
729        create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
730        create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
731        create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"),
732        create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
733        create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
734        create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"),
735        create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
736        create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
737        create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), };
738    List<KeyValueScanner> scanners = scanFixture(kvs);
739    try (StoreScanner scan =
740      new StoreScanner(new Scan().readAllVersions(), scanInfo, null, scanners)) {
741      List<Cell> results = new ArrayList<>();
742      assertEquals(true, scan.next(results));
743      assertEquals(0, results.size());
744      assertEquals(true, scan.next(results));
745      assertEquals(1, results.size());
746      assertEquals(kvs[kvs.length - 1], results.get(0));
747
748      assertEquals(false, scan.next(results));
749    }
750  }
751
752  @Test
753  public void testDeleteColumn() throws IOException {
754    KeyValue[] kvs =
755      new KeyValue[] { create("R1", "cf", "a", 10, KeyValue.Type.DeleteColumn, "dont-care"),
756        create("R1", "cf", "a", 9, KeyValue.Type.Delete, "dont-care"),
757        create("R1", "cf", "a", 8, KeyValue.Type.Put, "dont-care"),
758        create("R1", "cf", "b", 5, KeyValue.Type.Put, "dont-care") };
759    List<KeyValueScanner> scanners = scanFixture(kvs);
760    try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, null, scanners)) {
761      List<Cell> results = new ArrayList<>();
762      assertEquals(true, scan.next(results));
763      assertEquals(1, results.size());
764      assertEquals(kvs[3], results.get(0));
765    }
766  }
767
768  private static final KeyValue[] kvs =
769    new KeyValue[] { create("R1", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
770      create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
771      create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
772      create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
773      create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
774      create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
775      create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
776      create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
777      create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
778      create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), };
779
780  @Test
781  public void testSkipColumn() throws IOException {
782    List<KeyValueScanner> scanners = scanFixture(kvs);
783    try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) {
784      List<Cell> results = new ArrayList<>();
785      assertEquals(true, scan.next(results));
786      assertEquals(2, results.size());
787      assertEquals(kvs[0], results.get(0));
788      assertEquals(kvs[3], results.get(1));
789      results.clear();
790
791      assertEquals(true, scan.next(results));
792      assertEquals(1, results.size());
793      assertEquals(kvs[kvs.length - 1], results.get(0));
794
795      results.clear();
796      assertEquals(false, scan.next(results));
797    }
798  }
799
800  /*
801   * Test expiration of KeyValues in combination with a configured TTL for a column family (as
802   * should be triggered in a major compaction).
803   */
804  @Test
805  public void testWildCardTtlScan() throws IOException {
806    long now = EnvironmentEdgeManager.currentTime();
807    KeyValue[] kvs =
808      new KeyValue[] { create("R1", "cf", "a", now - 1000, KeyValue.Type.Put, "dont-care"),
809        create("R1", "cf", "b", now - 10, KeyValue.Type.Put, "dont-care"),
810        create("R1", "cf", "c", now - 200, KeyValue.Type.Put, "dont-care"),
811        create("R1", "cf", "d", now - 10000, KeyValue.Type.Put, "dont-care"),
812        create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
813        create("R2", "cf", "b", now - 10, KeyValue.Type.Put, "dont-care"),
814        create("R2", "cf", "c", now - 200, KeyValue.Type.Put, "dont-care"),
815        create("R2", "cf", "c", now - 1000, KeyValue.Type.Put, "dont-care") };
816    List<KeyValueScanner> scanners = scanFixture(kvs);
817    Scan scan = new Scan();
818    scan.readVersions(1);
819    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
820      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
821    try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
822      List<Cell> results = new ArrayList<>();
823      assertEquals(true, scanner.next(results));
824      assertEquals(2, results.size());
825      assertEquals(kvs[1], results.get(0));
826      assertEquals(kvs[2], results.get(1));
827      results.clear();
828
829      assertEquals(true, scanner.next(results));
830      assertEquals(3, results.size());
831      assertEquals(kvs[4], results.get(0));
832      assertEquals(kvs[5], results.get(1));
833      assertEquals(kvs[6], results.get(2));
834      results.clear();
835
836      assertEquals(false, scanner.next(results));
837    }
838  }
839
840  @Test
841  public void testScannerReseekDoesntNPE() throws Exception {
842    List<KeyValueScanner> scanners = scanFixture(kvs);
843    try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) {
844      // Previously a updateReaders twice in a row would cause an NPE. In test this would also
845      // normally cause an NPE because scan.store is null. So as long as we get through these
846      // two calls we are good and the bug was quashed.
847      scan.updateReaders(Collections.emptyList(), Collections.emptyList());
848      scan.updateReaders(Collections.emptyList(), Collections.emptyList());
849      scan.peek();
850    }
851  }
852
853  @Test
854  @Ignore("this fails, since we don't handle deletions, etc, in peek")
855  public void testPeek() throws Exception {
856    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
857      create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), };
858    List<KeyValueScanner> scanners = scanFixture(kvs);
859    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
860    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
861      assertNull(scan.peek());
862    }
863  }
864
865  /**
866   * Ensure that expired delete family markers don't override valid puts
867   */
868  @Test
869  public void testExpiredDeleteFamily() throws Exception {
870    long now = EnvironmentEdgeManager.currentTime();
871    KeyValue[] kvs = new KeyValue[] {
872      new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000,
873        KeyValue.Type.DeleteFamily),
874      create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), };
875    List<KeyValueScanner> scanners = scanFixture(kvs);
876    Scan scan = new Scan();
877    scan.readVersions(1);
878    // scanner with ttl equal to 500
879    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
880      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
881    try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
882      List<Cell> results = new ArrayList<>();
883      assertEquals(true, scanner.next(results));
884      assertEquals(1, results.size());
885      assertEquals(kvs[1], results.get(0));
886      results.clear();
887
888      assertEquals(false, scanner.next(results));
889    }
890  }
891
892  @Test
893  public void testDeleteMarkerLongevity() throws Exception {
894    try {
895      final long now = EnvironmentEdgeManager.currentTime();
896      EnvironmentEdgeManagerTestHelper.injectEdge(new EnvironmentEdge() {
897        @Override
898        public long currentTime() {
899          return now;
900        }
901      });
902      // @formatter:off
903      KeyValue[] kvs = new KeyValue[]{
904        /*0*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
905        now - 100, KeyValue.Type.DeleteFamily), // live
906        /*1*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
907        now - 1000, KeyValue.Type.DeleteFamily), // expired
908        /*2*/ create("R1", "cf", "a", now - 50,
909        KeyValue.Type.Put, "v3"), // live
910        /*3*/ create("R1", "cf", "a", now - 55,
911        KeyValue.Type.Delete, "dontcare"), // live
912        /*4*/ create("R1", "cf", "a", now - 55,
913        KeyValue.Type.Put, "deleted-version v2"), // deleted
914        /*5*/ create("R1", "cf", "a", now - 60,
915        KeyValue.Type.Put, "v1"), // live
916        /*6*/ create("R1", "cf", "a", now - 65,
917        KeyValue.Type.Put, "v0"), // max-version reached
918        /*7*/ create("R1", "cf", "a",
919        now - 100, KeyValue.Type.DeleteColumn, "dont-care"), // max-version
920        /*8*/ create("R1", "cf", "b", now - 600,
921        KeyValue.Type.DeleteColumn, "dont-care"), // expired
922        /*9*/ create("R1", "cf", "b", now - 70,
923        KeyValue.Type.Put, "v2"), // live
924        /*10*/ create("R1", "cf", "b", now - 750,
925        KeyValue.Type.Put, "v1"), // expired
926        /*11*/ create("R1", "cf", "c", now - 500,
927        KeyValue.Type.Delete, "dontcare"), // expired
928        /*12*/ create("R1", "cf", "c", now - 600,
929        KeyValue.Type.Put, "v1"), // expired
930        /*13*/ create("R1", "cf", "c", now - 1000,
931        KeyValue.Type.Delete, "dontcare"), // expired
932        /*14*/ create("R1", "cf", "d", now - 60,
933        KeyValue.Type.Put, "expired put"), // live
934        /*15*/ create("R1", "cf", "d", now - 100,
935        KeyValue.Type.Delete, "not-expired delete"), // live
936      };
937      // @formatter:on
938      List<KeyValueScanner> scanners = scanFixture(kvs);
939      ScanInfo scanInfo = new ScanInfo(CONF, Bytes.toBytes("cf"), 0 /* minVersions */,
940        2 /* maxVersions */, 500 /* ttl */, KeepDeletedCells.FALSE /* keepDeletedCells */,
941        HConstants.DEFAULT_BLOCKSIZE /* block size */, 200, /* timeToPurgeDeletes */
942        CellComparator.getInstance(), false);
943      try (StoreScanner scanner =
944        new StoreScanner(scanInfo, 2, ScanType.COMPACT_DROP_DELETES, scanners)) {
945        List<Cell> results = new ArrayList<>();
946        results = new ArrayList<>();
947        assertEquals(true, scanner.next(results));
948        assertEquals(kvs[0], results.get(0));
949        assertEquals(kvs[2], results.get(1));
950        assertEquals(kvs[3], results.get(2));
951        assertEquals(kvs[5], results.get(3));
952        assertEquals(kvs[9], results.get(4));
953        assertEquals(kvs[14], results.get(5));
954        assertEquals(kvs[15], results.get(6));
955        assertEquals(7, results.size());
956      }
957    } finally {
958      EnvironmentEdgeManagerTestHelper.reset();
959    }
960  }
961
962  @Test
963  public void testPreadNotEnabledForCompactionStoreScanners() throws Exception {
964    long now = EnvironmentEdgeManager.currentTime();
965    KeyValue[] kvs = new KeyValue[] {
966      new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000,
967        KeyValue.Type.DeleteFamily),
968      create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), };
969    List<KeyValueScanner> scanners = scanFixture(kvs);
970    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
971      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
972    try (StoreScanner storeScanner =
973      new StoreScanner(scanInfo, -1, ScanType.COMPACT_RETAIN_DELETES, scanners)) {
974      assertFalse(storeScanner.isScanUsePread());
975    }
976  }
977
978  @Test
979  public void testReadVersionWithRawAndFilter() throws IOException {
980    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, Long.MAX_VALUE, KeepDeletedCells.FALSE,
981      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
982    KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 3, KeyValue.Type.Put, "dont-care"),
983      create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
984      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care") };
985    List<KeyValueScanner> scanners = Arrays
986      .asList(new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), kvs) });
987
988    BinaryComparator comp = new BinaryComparator(Bytes.toBytes("a"));
989    Filter filter = new QualifierFilter(CompareOperator.EQUAL, comp);
990    Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2).setRaw(true);
991    scanSpec.setFilter(filter);
992    try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) {
993      List<Cell> results = new ArrayList<>();
994      assertEquals(true, scan.next(results));
995      assertEquals(2, results.size());
996    }
997  }
998
999  @Test
1000  public void testScannersClosedWhenCheckingOnlyMemStore() throws IOException {
1001    class MyCollectionBackedScanner extends CollectionBackedScanner {
1002      final boolean fileScanner;
1003      boolean closed;
1004
1005      MyCollectionBackedScanner(boolean fileScanner) {
1006        super(Collections.emptySortedSet());
1007        this.fileScanner = fileScanner;
1008      }
1009
1010      @Override
1011      public boolean isFileScanner() {
1012        return fileScanner;
1013      }
1014
1015      @Override
1016      public void close() {
1017        super.close();
1018        closed = true;
1019      }
1020    }
1021
1022    ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, Long.MAX_VALUE, KeepDeletedCells.FALSE,
1023      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
1024    InternalScan scan = new InternalScan(new Scan());
1025    scan.checkOnlyMemStore();
1026    MyCollectionBackedScanner fileScanner = new MyCollectionBackedScanner(true);
1027    MyCollectionBackedScanner memStoreScanner = new MyCollectionBackedScanner(false);
1028    List<? extends KeyValueScanner> allScanners = Arrays.asList(fileScanner, memStoreScanner);
1029
1030    try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, allScanners)) {
1031      List<KeyValueScanner> remaining = scanner.selectScannersFrom(null, allScanners);
1032
1033      assertEquals(1, remaining.size());
1034      assertSame(memStoreScanner, remaining.get(0));
1035      assertTrue(fileScanner.closed);
1036      assertFalse(memStoreScanner.closed);
1037    }
1038  }
1039}