Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.hamcrest.CoreMatchers.instanceOf;
021import static org.hamcrest.MatcherAssert.assertThat;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertFalse;
024import static org.junit.Assert.assertNotNull;
025import static org.junit.Assert.assertTrue;
026import static org.junit.Assert.fail;
027
028import java.io.IOException;
029import java.util.ArrayList;
030import java.util.Arrays;
031import java.util.Collection;
032import java.util.EnumSet;
033import java.util.HashSet;
034import java.util.List;
035import java.util.NavigableMap;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.Cell;
038import org.apache.hadoop.hbase.CellUtil;
039import org.apache.hadoop.hbase.ClusterMetrics.Option;
040import org.apache.hadoop.hbase.HBaseClassTestRule;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.KeyValue;
043import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
044import org.apache.hadoop.hbase.TableName;
045import org.apache.hadoop.hbase.TableNameTestRule;
046import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
047import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
048import org.apache.hadoop.hbase.testclassification.ClientTests;
049import org.apache.hadoop.hbase.testclassification.LargeTests;
050import org.apache.hadoop.hbase.util.Bytes;
051import org.junit.AfterClass;
052import org.junit.Assume;
053import org.junit.ClassRule;
054import org.junit.Rule;
055import org.junit.Test;
056import org.junit.experimental.categories.Category;
057import org.junit.runner.RunWith;
058import org.junit.runners.Parameterized;
059import org.slf4j.Logger;
060import org.slf4j.LoggerFactory;
061
062import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
063
064/**
065 * Run tests that use the HBase clients; {@link Table}. Sets up the HBase mini cluster once at start
066 * and runs through all client tests. Each creates a table named for the method and does its stuff
067 * against that. Parameterized to run with different registry implementations.
068 */
069@Category({ LargeTests.class, ClientTests.class })
070@SuppressWarnings("deprecation")
071@RunWith(Parameterized.class)
072public class TestFromClientSide4 extends FromClientSideBase {
073  private static final Logger LOG = LoggerFactory.getLogger(TestFromClientSide4.class);
074  @ClassRule
075  public static final HBaseClassTestRule CLASS_RULE =
076    HBaseClassTestRule.forClass(TestFromClientSide4.class);
077  @Rule
078  public TableNameTestRule name = new TableNameTestRule();
079
080  // To keep the child classes happy.
081  TestFromClientSide4() {
082  }
083
084  public TestFromClientSide4(Class registry, int numHedgedReqs) throws Exception {
085    initialize(registry, numHedgedReqs, MultiRowMutationEndpoint.class);
086  }
087
088  @Parameterized.Parameters
089  public static Collection parameters() {
090    return Arrays.asList(new Object[][] { { MasterRegistry.class, 1 }, { MasterRegistry.class, 2 },
091      { ZKConnectionRegistry.class, 1 } });
092  }
093
094  @AfterClass
095  public static void tearDownAfterClass() throws Exception {
096    afterClass();
097  }
098
099  /**
100   * Test batch operations with combination of valid and invalid args
101   */
102  @Test
103  public void testBatchOperationsWithErrors() throws Exception {
104    final TableName tableName = name.getTableName();
105    try (Table foo = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 10)) {
106
107      int NUM_OPS = 100;
108
109      // 1.1 Put with no column families (local validation, runtime exception)
110      List<Put> puts = new ArrayList<>(NUM_OPS);
111      for (int i = 0; i != NUM_OPS; i++) {
112        Put put = new Put(Bytes.toBytes(i));
113        puts.add(put);
114      }
115
116      try {
117        foo.put(puts);
118        fail();
119      } catch (IllegalArgumentException e) {
120        // expected
121        assertEquals(NUM_OPS, puts.size());
122      }
123
124      // 1.2 Put with invalid column family
125      puts.clear();
126      for (int i = 0; i < NUM_OPS; i++) {
127        Put put = new Put(Bytes.toBytes(i));
128        put.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY, Bytes.toBytes(i));
129        puts.add(put);
130      }
131
132      try {
133        foo.put(puts);
134        fail();
135      } catch (RetriesExhaustedException e) {
136        // expected
137        assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
138      }
139
140      // 2.1 Get non-existent rows
141      List<Get> gets = new ArrayList<>(NUM_OPS);
142      for (int i = 0; i < NUM_OPS; i++) {
143        Get get = new Get(Bytes.toBytes(i));
144        gets.add(get);
145      }
146      Result[] getsResult = foo.get(gets);
147      assertNotNull(getsResult);
148      assertEquals(NUM_OPS, getsResult.length);
149      for (int i = 0; i < NUM_OPS; i++) {
150        Result getResult = getsResult[i];
151        if (i % 2 == 0) {
152          assertFalse(getResult.isEmpty());
153        } else {
154          assertTrue(getResult.isEmpty());
155        }
156      }
157
158      // 2.2 Get with invalid column family
159      gets.clear();
160      for (int i = 0; i < NUM_OPS; i++) {
161        Get get = new Get(Bytes.toBytes(i));
162        get.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
163        gets.add(get);
164      }
165      try {
166        foo.get(gets);
167        fail();
168      } catch (RetriesExhaustedException e) {
169        // expected
170        assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
171      }
172
173      // 3.1 Delete with invalid column family
174      List<Delete> deletes = new ArrayList<>(NUM_OPS);
175      for (int i = 0; i < NUM_OPS; i++) {
176        Delete delete = new Delete(Bytes.toBytes(i));
177        delete.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
178        deletes.add(delete);
179      }
180      try {
181        foo.delete(deletes);
182        fail();
183      } catch (RetriesExhaustedException e) {
184        // expected
185        assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
186      }
187
188      // all valid rows should have been deleted
189      gets.clear();
190      for (int i = 0; i < NUM_OPS; i++) {
191        Get get = new Get(Bytes.toBytes(i));
192        gets.add(get);
193      }
194      getsResult = foo.get(gets);
195      assertNotNull(getsResult);
196      assertEquals(NUM_OPS, getsResult.length);
197      for (Result getResult : getsResult) {
198        assertTrue(getResult.isEmpty());
199      }
200
201      // 3.2 Delete non-existent rows
202      deletes.clear();
203      for (int i = 0; i < NUM_OPS; i++) {
204        Delete delete = new Delete(Bytes.toBytes(i));
205        deletes.add(delete);
206      }
207      foo.delete(deletes);
208    }
209  }
210
211  //
212  // JIRA Testers
213  //
214
215  /**
216   * HBASE-867 If millions of columns in a column family, hbase scanner won't come up Test will
217   * create numRows rows, each with numColsPerRow columns (1 version each), and attempt to scan them
218   * all. To test at scale, up numColsPerRow to the millions (have not gotten that to work running
219   * as junit though)
220   */
221  @Test
222  public void testJiraTest867() throws Exception {
223    int numRows = 10;
224    int numColsPerRow = 2000;
225
226    final TableName tableName = name.getTableName();
227
228    byte[][] ROWS = makeN(ROW, numRows);
229    byte[][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
230
231    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
232
233      // Insert rows
234
235      for (int i = 0; i < numRows; i++) {
236        Put put = new Put(ROWS[i]);
237        put.setDurability(Durability.SKIP_WAL);
238        for (int j = 0; j < numColsPerRow; j++) {
239          put.addColumn(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
240        }
241        assertEquals("Put expected to contain " + numColsPerRow + " columns but " + "only contains "
242          + put.size(), put.size(), numColsPerRow);
243        ht.put(put);
244      }
245
246      // Get a row
247      Get get = new Get(ROWS[numRows - 1]);
248      Result result = ht.get(get);
249      assertNumKeys(result, numColsPerRow);
250      Cell[] keys = result.rawCells();
251      for (int i = 0; i < result.size(); i++) {
252        assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
253      }
254
255      // Scan the rows
256      Scan scan = new Scan();
257      try (ResultScanner scanner = ht.getScanner(scan)) {
258        int rowCount = 0;
259        while ((result = scanner.next()) != null) {
260          assertNumKeys(result, numColsPerRow);
261          Cell[] kvs = result.rawCells();
262          for (int i = 0; i < numColsPerRow; i++) {
263            assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
264          }
265          rowCount++;
266        }
267        assertEquals(
268          "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows",
269          rowCount, numRows);
270      }
271
272      // flush and try again
273
274      TEST_UTIL.flush();
275
276      // Get a row
277      get = new Get(ROWS[numRows - 1]);
278      result = ht.get(get);
279      assertNumKeys(result, numColsPerRow);
280      keys = result.rawCells();
281      for (int i = 0; i < result.size(); i++) {
282        assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
283      }
284
285      // Scan the rows
286      scan = new Scan();
287      try (ResultScanner scanner = ht.getScanner(scan)) {
288        int rowCount = 0;
289        while ((result = scanner.next()) != null) {
290          assertNumKeys(result, numColsPerRow);
291          Cell[] kvs = result.rawCells();
292          for (int i = 0; i < numColsPerRow; i++) {
293            assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
294          }
295          rowCount++;
296        }
297        assertEquals(
298          "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows",
299          rowCount, numRows);
300      }
301    }
302  }
303
304  /**
305   * HBASE-861 get with timestamp will return a value if there is a version with an earlier
306   * timestamp
307   */
308  @Test
309  public void testJiraTest861() throws Exception {
310    final TableName tableName = name.getTableName();
311    byte[][] VALUES = makeNAscii(VALUE, 7);
312    long[] STAMPS = makeStamps(7);
313
314    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
315
316      // Insert three versions
317
318      Put put = new Put(ROW);
319      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
320      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
321      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
322      ht.put(put);
323
324      // Get the middle value
325      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
326
327      // Try to get one version before (expect fail)
328      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
329
330      // Try to get one version after (expect fail)
331      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
332
333      // Try same from storefile
334      TEST_UTIL.flush();
335      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
336      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
337      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
338
339      // Insert two more versions surrounding others, into memstore
340      put = new Put(ROW);
341      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
342      put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
343      ht.put(put);
344
345      // Check we can get everything we should and can't get what we shouldn't
346      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
347      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
348      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
349      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
350      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
351      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
352      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
353
354      // Try same from two storefiles
355      TEST_UTIL.flush();
356      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
357      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
358      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
359      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
360      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
361      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
362      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
363    }
364  }
365
366  /**
367   * HBASE-33 Add a HTable get/obtainScanner method that retrieves all versions of a particular
368   * column and row between two timestamps
369   */
370  @Test
371  public void testJiraTest33() throws Exception {
372    final TableName tableName = name.getTableName();
373    byte[][] VALUES = makeNAscii(VALUE, 7);
374    long[] STAMPS = makeStamps(7);
375
376    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
377
378      // Insert lots versions
379
380      Put put = new Put(ROW);
381      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
382      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
383      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
384      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
385      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
386      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
387      ht.put(put);
388
389      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
390      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
391      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
392      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
393
394      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
395      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
396      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
397      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
398
399      // Try same from storefile
400      TEST_UTIL.flush();
401
402      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
403      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
404      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
405      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
406
407      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
408      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
409      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
410      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
411    }
412  }
413
414  /**
415   * HBASE-1014 commit(BatchUpdate) method should return timestamp
416   */
417  @Test
418  public void testJiraTest1014() throws Exception {
419    final TableName tableName = name.getTableName();
420
421    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
422
423      long manualStamp = 12345;
424
425      // Insert lots versions
426
427      Put put = new Put(ROW);
428      put.addColumn(FAMILY, QUALIFIER, manualStamp, VALUE);
429      ht.put(put);
430
431      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
432      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp - 1);
433      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp + 1);
434    }
435  }
436
437  /**
438   * HBASE-1182 Scan for columns > some timestamp
439   */
440  @Test
441  public void testJiraTest1182() throws Exception {
442    final TableName tableName = name.getTableName();
443    byte[][] VALUES = makeNAscii(VALUE, 7);
444    long[] STAMPS = makeStamps(7);
445
446    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
447
448      // Insert lots versions
449
450      Put put = new Put(ROW);
451      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
452      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
453      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
454      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
455      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
456      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
457      ht.put(put);
458
459      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
460      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
461      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
462
463      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
464      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
465      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
466
467      // Try same from storefile
468      TEST_UTIL.flush();
469
470      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
471      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
472      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
473
474      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
475      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
476      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
477    }
478  }
479
480  /**
481   * HBASE-52 Add a means of scanning over all versions
482   */
483  @Test
484  public void testJiraTest52() throws Exception {
485    final TableName tableName = name.getTableName();
486    byte[][] VALUES = makeNAscii(VALUE, 7);
487    long[] STAMPS = makeStamps(7);
488
489    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
490
491      // Insert lots versions
492
493      Put put = new Put(ROW);
494      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
495      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
496      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
497      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
498      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
499      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
500      ht.put(put);
501
502      getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
503
504      scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
505
506      // Try same from storefile
507      TEST_UTIL.flush();
508
509      getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
510
511      scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
512    }
513  }
514
515  @Test
516  @SuppressWarnings("checkstyle:MethodLength")
517  public void testDuplicateVersions() throws Exception {
518    final TableName tableName = name.getTableName();
519
520    long[] STAMPS = makeStamps(20);
521    byte[][] VALUES = makeNAscii(VALUE, 20);
522
523    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
524
525      // Insert 4 versions of same column
526      Put put = new Put(ROW);
527      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
528      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
529      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
530      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
531      ht.put(put);
532
533      // Verify we can get each one properly
534      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
535      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
536      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
537      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
538      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
539      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
540      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
541      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
542
543      // Verify we don't accidentally get others
544      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
545      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
546      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
547      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
548      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
549      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
550
551      // Ensure maxVersions in query is respected
552      Get get = new Get(ROW);
553      get.addColumn(FAMILY, QUALIFIER);
554      get.readVersions(2);
555      Result result = ht.get(get);
556      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
557        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
558
559      Scan scan = new Scan().withStartRow(ROW);
560      scan.addColumn(FAMILY, QUALIFIER);
561      scan.readVersions(2);
562      result = getSingleScanResult(ht, scan);
563      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
564        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
565
566      // Flush and redo
567
568      TEST_UTIL.flush();
569
570      // Verify we can get each one properly
571      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
572      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
573      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
574      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
575      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
576      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
577      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
578      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
579
580      // Verify we don't accidentally get others
581      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
582      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
583      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
584      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
585      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
586      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
587
588      // Ensure maxVersions in query is respected
589      get = new Get(ROW);
590      get.addColumn(FAMILY, QUALIFIER);
591      get.readVersions(2);
592      result = ht.get(get);
593      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
594        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
595
596      scan = new Scan().withStartRow(ROW);
597      scan.addColumn(FAMILY, QUALIFIER);
598      scan.readVersions(2);
599      result = getSingleScanResult(ht, scan);
600      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
601        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
602
603      // Add some memstore and retest
604
605      // Insert 4 more versions of same column and a dupe
606      put = new Put(ROW);
607      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
608      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
609      put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
610      put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
611      put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
612      ht.put(put);
613
614      // Ensure maxVersions in query is respected
615      get = new Get(ROW);
616      get.addColumn(FAMILY, QUALIFIER);
617      get.readVersions(7);
618      result = ht.get(get);
619      assertNResult(result, ROW, FAMILY, QUALIFIER,
620        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
621        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
622          VALUES[8] },
623        0, 6);
624
625      scan = new Scan().withStartRow(ROW);
626      scan.addColumn(FAMILY, QUALIFIER);
627      scan.readVersions(7);
628      result = getSingleScanResult(ht, scan);
629      assertNResult(result, ROW, FAMILY, QUALIFIER,
630        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
631        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
632          VALUES[8] },
633        0, 6);
634
635      get = new Get(ROW);
636      get.readVersions(7);
637      result = ht.get(get);
638      assertNResult(result, ROW, FAMILY, QUALIFIER,
639        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
640        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
641          VALUES[8] },
642        0, 6);
643
644      scan = new Scan().withStartRow(ROW);
645      scan.readVersions(7);
646      result = getSingleScanResult(ht, scan);
647      assertNResult(result, ROW, FAMILY, QUALIFIER,
648        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
649        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
650          VALUES[8] },
651        0, 6);
652
653      // Verify we can get each one properly
654      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
655      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
656      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
657      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
658      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
659      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
660      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
661      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
662
663      // Verify we don't accidentally get others
664      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
665      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
666      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
667      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
668
669      // Ensure maxVersions of table is respected
670
671      TEST_UTIL.flush();
672
673      // Insert 4 more versions of same column and a dupe
674      put = new Put(ROW);
675      put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
676      put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
677      put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
678      put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
679      ht.put(put);
680
681      get = new Get(ROW);
682      get.addColumn(FAMILY, QUALIFIER);
683      get.readVersions(Integer.MAX_VALUE);
684      result = ht.get(get);
685      assertNResult(result, ROW, FAMILY, QUALIFIER,
686        new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
687          STAMPS[11], STAMPS[13], STAMPS[15] },
688        new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
689          VALUES[11], VALUES[13], VALUES[15] },
690        0, 9);
691
692      scan = new Scan().withStartRow(ROW);
693      scan.addColumn(FAMILY, QUALIFIER);
694      scan.readVersions(Integer.MAX_VALUE);
695      result = getSingleScanResult(ht, scan);
696      assertNResult(result, ROW, FAMILY, QUALIFIER,
697        new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
698          STAMPS[11], STAMPS[13], STAMPS[15] },
699        new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
700          VALUES[11], VALUES[13], VALUES[15] },
701        0, 9);
702
703      // Delete a version in the memstore and a version in a storefile
704      Delete delete = new Delete(ROW);
705      delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]);
706      delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]);
707      ht.delete(delete);
708
709      // Test that it's gone
710      get = new Get(ROW);
711      get.addColumn(FAMILY, QUALIFIER);
712      get.readVersions(Integer.MAX_VALUE);
713      result = ht.get(get);
714      assertNResult(result, ROW, FAMILY, QUALIFIER,
715        new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
716          STAMPS[9], STAMPS[13], STAMPS[15] },
717        new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
718          VALUES[9], VALUES[13], VALUES[15] },
719        0, 9);
720
721      scan = new Scan().withStartRow(ROW);
722      scan.addColumn(FAMILY, QUALIFIER);
723      scan.readVersions(Integer.MAX_VALUE);
724      result = getSingleScanResult(ht, scan);
725      assertNResult(result, ROW, FAMILY, QUALIFIER,
726        new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
727          STAMPS[9], STAMPS[13], STAMPS[15] },
728        new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
729          VALUES[9], VALUES[13], VALUES[15] },
730        0, 9);
731    }
732  }
733
734  @Test
735  public void testUpdates() throws Exception {
736    final TableName tableName = name.getTableName();
737    try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
738
739      // Write a column with values at timestamp 1, 2 and 3
740      byte[] row = Bytes.toBytes("row1");
741      byte[] qualifier = Bytes.toBytes("myCol");
742      Put put = new Put(row);
743      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
744      hTable.put(put);
745
746      put = new Put(row);
747      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
748      hTable.put(put);
749
750      put = new Put(row);
751      put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
752      hTable.put(put);
753
754      Get get = new Get(row);
755      get.addColumn(FAMILY, qualifier);
756      get.readAllVersions();
757
758      // Check that the column indeed has the right values at timestamps 1 and
759      // 2
760      Result result = hTable.get(get);
761      NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
762      assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
763      assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
764
765      // Update the value at timestamp 1
766      put = new Put(row);
767      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
768      hTable.put(put);
769
770      // Update the value at timestamp 2
771      put = new Put(row);
772      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
773      hTable.put(put);
774
775      // Check that the values at timestamp 2 and 1 got updated
776      result = hTable.get(get);
777      navigableMap = result.getMap().get(FAMILY).get(qualifier);
778      assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
779      assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
780    }
781  }
782
783  @Test
784  public void testUpdatesWithMajorCompaction() throws Exception {
785    final TableName tableName = name.getTableName();
786    try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
787      Admin admin = TEST_UTIL.getAdmin()) {
788
789      // Write a column with values at timestamp 1, 2 and 3
790      byte[] row = Bytes.toBytes("row2");
791      byte[] qualifier = Bytes.toBytes("myCol");
792      Put put = new Put(row);
793      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
794      hTable.put(put);
795
796      put = new Put(row);
797      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
798      hTable.put(put);
799
800      put = new Put(row);
801      put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
802      hTable.put(put);
803
804      Get get = new Get(row);
805      get.addColumn(FAMILY, qualifier);
806      get.readAllVersions();
807
808      // Check that the column indeed has the right values at timestamps 1 and
809      // 2
810      Result result = hTable.get(get);
811      NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
812      assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
813      assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
814
815      // Trigger a major compaction
816      admin.flush(tableName);
817      admin.majorCompact(tableName);
818      Thread.sleep(6000);
819
820      // Update the value at timestamp 1
821      put = new Put(row);
822      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
823      hTable.put(put);
824
825      // Update the value at timestamp 2
826      put = new Put(row);
827      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
828      hTable.put(put);
829
830      // Trigger a major compaction
831      admin.flush(tableName);
832      admin.majorCompact(tableName);
833      Thread.sleep(6000);
834
835      // Check that the values at timestamp 2 and 1 got updated
836      result = hTable.get(get);
837      navigableMap = result.getMap().get(FAMILY).get(qualifier);
838      assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
839      assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
840    }
841  }
842
843  @Test
844  public void testMajorCompactionBetweenTwoUpdates() throws Exception {
845    final TableName tableName = name.getTableName();
846    try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
847      Admin admin = TEST_UTIL.getAdmin()) {
848
849      // Write a column with values at timestamp 1, 2 and 3
850      byte[] row = Bytes.toBytes("row3");
851      byte[] qualifier = Bytes.toBytes("myCol");
852      Put put = new Put(row);
853      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
854      hTable.put(put);
855
856      put = new Put(row);
857      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
858      hTable.put(put);
859
860      put = new Put(row);
861      put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
862      hTable.put(put);
863
864      Get get = new Get(row);
865      get.addColumn(FAMILY, qualifier);
866      get.readAllVersions();
867
868      // Check that the column indeed has the right values at timestamps 1 and
869      // 2
870      Result result = hTable.get(get);
871      NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
872      assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
873      assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
874
875      // Trigger a major compaction
876      admin.flush(tableName);
877      admin.majorCompact(tableName);
878      Thread.sleep(6000);
879
880      // Update the value at timestamp 1
881      put = new Put(row);
882      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
883      hTable.put(put);
884
885      // Trigger a major compaction
886      admin.flush(tableName);
887      admin.majorCompact(tableName);
888      Thread.sleep(6000);
889
890      // Update the value at timestamp 2
891      put = new Put(row);
892      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
893      hTable.put(put);
894
895      // Trigger a major compaction
896      admin.flush(tableName);
897      admin.majorCompact(tableName);
898      Thread.sleep(6000);
899
900      // Check that the values at timestamp 2 and 1 got updated
901      result = hTable.get(get);
902      navigableMap = result.getMap().get(FAMILY).get(qualifier);
903
904      assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
905      assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
906    }
907  }
908
909  @Test
910  public void testGet_EmptyTable() throws IOException {
911    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
912      Get get = new Get(ROW);
913      get.addFamily(FAMILY);
914      Result r = table.get(get);
915      assertTrue(r.isEmpty());
916    }
917  }
918
919  @Test
920  public void testGet_NullQualifier() throws IOException {
921    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
922      Put put = new Put(ROW);
923      put.addColumn(FAMILY, QUALIFIER, VALUE);
924      table.put(put);
925
926      put = new Put(ROW);
927      put.addColumn(FAMILY, null, VALUE);
928      table.put(put);
929      LOG.info("Row put");
930
931      Get get = new Get(ROW);
932      get.addColumn(FAMILY, null);
933      Result r = table.get(get);
934      assertEquals(1, r.size());
935
936      get = new Get(ROW);
937      get.addFamily(FAMILY);
938      r = table.get(get);
939      assertEquals(2, r.size());
940    }
941  }
942
943  @Test
944  public void testGet_NonExistentRow() throws IOException {
945    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
946      Put put = new Put(ROW);
947      put.addColumn(FAMILY, QUALIFIER, VALUE);
948      table.put(put);
949      LOG.info("Row put");
950
951      Get get = new Get(ROW);
952      get.addFamily(FAMILY);
953      Result r = table.get(get);
954      assertFalse(r.isEmpty());
955      System.out.println("Row retrieved successfully");
956
957      byte[] missingrow = Bytes.toBytes("missingrow");
958      get = new Get(missingrow);
959      get.addFamily(FAMILY);
960      r = table.get(get);
961      assertTrue(r.isEmpty());
962      LOG.info("Row missing as it should be");
963    }
964  }
965
966  @Test
967  public void testPut() throws IOException {
968    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
969    final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
970    final byte[] row1 = Bytes.toBytes("row1");
971    final byte[] row2 = Bytes.toBytes("row2");
972    final byte[] value = Bytes.toBytes("abcd");
973    try (Table table =
974      TEST_UTIL.createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
975      Put put = new Put(row1);
976      put.addColumn(CONTENTS_FAMILY, null, value);
977      table.put(put);
978
979      put = new Put(row2);
980      put.addColumn(CONTENTS_FAMILY, null, value);
981
982      assertEquals(1, put.size());
983      assertEquals(1, put.getFamilyCellMap().get(CONTENTS_FAMILY).size());
984
985      // KeyValue v1 expectation. Cast for now until we go all Cell all the time. TODO
986      KeyValue kv = (KeyValue) put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);
987
988      assertTrue(Bytes.equals(CellUtil.cloneFamily(kv), CONTENTS_FAMILY));
989      // will it return null or an empty byte array?
990      assertTrue(Bytes.equals(CellUtil.cloneQualifier(kv), new byte[0]));
991
992      assertTrue(Bytes.equals(CellUtil.cloneValue(kv), value));
993
994      table.put(put);
995
996      Scan scan = new Scan();
997      scan.addColumn(CONTENTS_FAMILY, null);
998      try (ResultScanner scanner = table.getScanner(scan)) {
999        for (Result r : scanner) {
1000          for (Cell key : r.rawCells()) {
1001            System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
1002          }
1003        }
1004      }
1005    }
1006  }
1007
1008  @Test
1009  public void testPutNoCF() throws IOException {
1010    final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
1011    final byte[] VAL = Bytes.toBytes(100);
1012    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
1013      boolean caughtNSCFE = false;
1014
1015      try {
1016        Put p = new Put(ROW);
1017        p.addColumn(BAD_FAM, QUALIFIER, VAL);
1018        table.put(p);
1019      } catch (Exception e) {
1020        caughtNSCFE = e instanceof NoSuchColumnFamilyException;
1021      }
1022      assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);
1023    }
1024  }
1025
1026  @Test
1027  public void testRowsPut() throws IOException {
1028    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1029    final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
1030    final int NB_BATCH_ROWS = 10;
1031    final byte[] value = Bytes.toBytes("abcd");
1032    try (Table table =
1033      TEST_UTIL.createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
1034      ArrayList<Put> rowsUpdate = new ArrayList<>();
1035      for (int i = 0; i < NB_BATCH_ROWS; i++) {
1036        byte[] row = Bytes.toBytes("row" + i);
1037        Put put = new Put(row);
1038        put.setDurability(Durability.SKIP_WAL);
1039        put.addColumn(CONTENTS_FAMILY, null, value);
1040        rowsUpdate.add(put);
1041      }
1042      table.put(rowsUpdate);
1043      Scan scan = new Scan();
1044      scan.addFamily(CONTENTS_FAMILY);
1045      try (ResultScanner scanner = table.getScanner(scan)) {
1046        int nbRows = Iterables.size(scanner);
1047        assertEquals(NB_BATCH_ROWS, nbRows);
1048      }
1049    }
1050  }
1051
1052  @Test
1053  public void testRowsPutBufferedManyManyFlushes() throws IOException {
1054    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1055    final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
1056    final byte[] value = Bytes.toBytes("abcd");
1057    final int NB_BATCH_ROWS = 10;
1058    try (Table table =
1059      TEST_UTIL.createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
1060      ArrayList<Put> rowsUpdate = new ArrayList<>();
1061      for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
1062        byte[] row = Bytes.toBytes("row" + i);
1063        Put put = new Put(row);
1064        put.setDurability(Durability.SKIP_WAL);
1065        put.addColumn(CONTENTS_FAMILY, null, value);
1066        rowsUpdate.add(put);
1067      }
1068      table.put(rowsUpdate);
1069
1070      Scan scan = new Scan();
1071      scan.addFamily(CONTENTS_FAMILY);
1072      try (ResultScanner scanner = table.getScanner(scan)) {
1073        int nbRows = Iterables.size(scanner);
1074        assertEquals(NB_BATCH_ROWS * 10, nbRows);
1075      }
1076    }
1077  }
1078
1079  @Test
1080  public void testAddKeyValue() {
1081    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1082    final byte[] value = Bytes.toBytes("abcd");
1083    final byte[] row1 = Bytes.toBytes("row1");
1084    final byte[] row2 = Bytes.toBytes("row2");
1085    byte[] qualifier = Bytes.toBytes("qf1");
1086    Put put = new Put(row1);
1087
1088    // Adding KeyValue with the same row
1089    KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
1090    boolean ok = true;
1091    try {
1092      put.add(kv);
1093    } catch (IOException e) {
1094      ok = false;
1095    }
1096    assertTrue(ok);
1097
1098    // Adding KeyValue with the different row
1099    kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
1100    ok = false;
1101    try {
1102      put.add(kv);
1103    } catch (IOException e) {
1104      ok = true;
1105    }
1106    assertTrue(ok);
1107  }
1108
1109  /**
1110   * test for HBASE-737
1111   */
1112  @Test
1113  public void testHBase737() throws IOException {
1114    final byte[] FAM1 = Bytes.toBytes("fam1");
1115    final byte[] FAM2 = Bytes.toBytes("fam2");
1116    // Open table
1117    try (Table table = TEST_UTIL.createTable(name.getTableName(), new byte[][] { FAM1, FAM2 })) {
1118      // Insert some values
1119      Put put = new Put(ROW);
1120      put.addColumn(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
1121      table.put(put);
1122      try {
1123        Thread.sleep(1000);
1124      } catch (InterruptedException i) {
1125        // ignore
1126      }
1127
1128      put = new Put(ROW);
1129      put.addColumn(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
1130      table.put(put);
1131
1132      try {
1133        Thread.sleep(1000);
1134      } catch (InterruptedException i) {
1135        // ignore
1136      }
1137
1138      put = new Put(ROW);
1139      put.addColumn(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
1140      table.put(put);
1141
1142      long[] times = new long[3];
1143
1144      // First scan the memstore
1145
1146      Scan scan = new Scan();
1147      scan.addFamily(FAM1);
1148      scan.addFamily(FAM2);
1149      try (ResultScanner s = table.getScanner(scan)) {
1150        int index = 0;
1151        Result r;
1152        while ((r = s.next()) != null) {
1153          for (Cell key : r.rawCells()) {
1154            times[index++] = key.getTimestamp();
1155          }
1156        }
1157      }
1158      for (int i = 0; i < times.length - 1; i++) {
1159        for (int j = i + 1; j < times.length; j++) {
1160          assertTrue(times[j] > times[i]);
1161        }
1162      }
1163
1164      // Flush data to disk and try again
1165      TEST_UTIL.flush();
1166
1167      // Reset times
1168      Arrays.fill(times, 0);
1169
1170      try {
1171        Thread.sleep(1000);
1172      } catch (InterruptedException i) {
1173        // ignore
1174      }
1175      scan = new Scan();
1176      scan.addFamily(FAM1);
1177      scan.addFamily(FAM2);
1178      try (ResultScanner s = table.getScanner(scan)) {
1179        int index = 0;
1180        Result r = null;
1181        while ((r = s.next()) != null) {
1182          for (Cell key : r.rawCells()) {
1183            times[index++] = key.getTimestamp();
1184          }
1185        }
1186        for (int i = 0; i < times.length - 1; i++) {
1187          for (int j = i + 1; j < times.length; j++) {
1188            assertTrue(times[j] > times[i]);
1189          }
1190        }
1191      }
1192    }
1193  }
1194
1195  @Test
1196  public void testListTables() throws IOException {
1197    final String testTableName = name.getTableName().toString();
1198    final TableName tableName1 = TableName.valueOf(testTableName + "1");
1199    final TableName tableName2 = TableName.valueOf(testTableName + "2");
1200    final TableName tableName3 = TableName.valueOf(testTableName + "3");
1201    TableName[] tables = new TableName[] { tableName1, tableName2, tableName3 };
1202    for (TableName table : tables) {
1203      TEST_UTIL.createTable(table, FAMILY);
1204    }
1205    try (Admin admin = TEST_UTIL.getAdmin()) {
1206      List<TableDescriptor> ts = admin.listTableDescriptors();
1207      HashSet<TableDescriptor> result = new HashSet<>(ts);
1208      int size = result.size();
1209      assertTrue(size >= tables.length);
1210      for (TableName table : tables) {
1211        boolean found = false;
1212        for (TableDescriptor t : ts) {
1213          if (t.getTableName().equals(table)) {
1214            found = true;
1215            break;
1216          }
1217        }
1218        assertTrue("Not found: " + table, found);
1219      }
1220    }
1221  }
1222
1223  /**
1224   * simple test that just executes parts of the client API that accept a pre-created Connection
1225   * instance
1226   */
1227  @Test
1228  public void testUnmanagedHConnection() throws IOException {
1229    final TableName tableName = name.getTableName();
1230    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
1231    try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
1232      Table t = conn.getTable(tableName); Admin admin = conn.getAdmin()) {
1233      assertTrue(admin.tableExists(tableName));
1234      assertTrue(t.get(new Get(ROW)).isEmpty());
1235    }
1236  }
1237
1238  /**
1239   * test of that unmanaged HConnections are able to reconnect properly (see HBASE-5058)
1240   */
1241  @Test
1242  public void testUnmanagedHConnectionReconnect() throws Exception {
1243    Configuration conf = TEST_UTIL.getConfiguration();
1244    Class registryImpl = conf.getClass(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY,
1245      ZKConnectionRegistry.class);
1246    // This test does not make sense for MasterRegistry since it stops the only master in the
1247    // cluster and starts a new master without populating the underlying config for the connection.
1248    Assume.assumeFalse(registryImpl.equals(MasterRegistry.class));
1249    final TableName tableName = name.getTableName();
1250    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
1251    try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
1252      try (Table t = conn.getTable(tableName); Admin admin = conn.getAdmin()) {
1253        assertTrue(admin.tableExists(tableName));
1254        assertTrue(t.get(new Get(ROW)).isEmpty());
1255      }
1256
1257      // stop the master
1258      SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1259      cluster.stopMaster(0, false);
1260      cluster.waitOnMaster(0);
1261
1262      // start up a new master
1263      cluster.startMaster();
1264      assertTrue(cluster.waitForActiveAndReadyMaster());
1265
1266      // test that the same unmanaged connection works with a new
1267      // Admin and can connect to the new master;
1268      try (Admin admin = conn.getAdmin()) {
1269        assertTrue(admin.tableExists(tableName));
1270        assertEquals(
1271          admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size(),
1272          SLAVES);
1273      }
1274    }
1275  }
1276
1277  @Test
1278  public void testMiscHTableStuff() throws IOException {
1279    final String testTableName = name.getTableName().toString();
1280    final TableName tableAname = TableName.valueOf(testTableName + "A");
1281    final TableName tableBname = TableName.valueOf(testTableName + "B");
1282    final byte[] attrName = Bytes.toBytes("TESTATTR");
1283    final byte[] attrValue = Bytes.toBytes("somevalue");
1284    byte[] value = Bytes.toBytes("value");
1285
1286    try (Table a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
1287      Table b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY)) {
1288      Put put = new Put(ROW);
1289      put.addColumn(HConstants.CATALOG_FAMILY, null, value);
1290      a.put(put);
1291
1292      // open a new connection to A and a connection to b
1293      try (Table newA = TEST_UTIL.getConnection().getTable(tableAname)) {
1294
1295        // copy data from A to B
1296        Scan scan = new Scan();
1297        scan.addFamily(HConstants.CATALOG_FAMILY);
1298        try (ResultScanner s = newA.getScanner(scan)) {
1299          for (Result r : s) {
1300            put = new Put(r.getRow());
1301            put.setDurability(Durability.SKIP_WAL);
1302            for (Cell kv : r.rawCells()) {
1303              put.add(kv);
1304            }
1305            b.put(put);
1306          }
1307        }
1308      }
1309
1310      // Opening a new connection to A will cause the tables to be reloaded
1311      try (Table anotherA = TEST_UTIL.getConnection().getTable(tableAname)) {
1312        Get get = new Get(ROW);
1313        get.addFamily(HConstants.CATALOG_FAMILY);
1314        anotherA.get(get);
1315      }
1316
1317      // We can still access A through newA because it has the table information
1318      // cached. And if it needs to recalibrate, that will cause the information
1319      // to be reloaded.
1320
1321      // Test user metadata
1322      Admin admin = TEST_UTIL.getAdmin();
1323      // make a modifiable descriptor
1324      TableDescriptor desc = a.getDescriptor();
1325      // offline the table
1326      admin.disableTable(tableAname);
1327      // add a user attribute to HTD
1328      TableDescriptorBuilder builder =
1329        TableDescriptorBuilder.newBuilder(desc).setValue(attrName, attrValue);
1330      // add a user attribute to HCD
1331      for (ColumnFamilyDescriptor c : desc.getColumnFamilies()) {
1332        builder.modifyColumnFamily(
1333          ColumnFamilyDescriptorBuilder.newBuilder(c).setValue(attrName, attrValue).build());
1334      }
1335      // update metadata for all regions of this table
1336      admin.modifyTable(builder.build());
1337      // enable the table
1338      admin.enableTable(tableAname);
1339
1340      // Test that attribute changes were applied
1341      desc = a.getDescriptor();
1342      assertEquals("wrong table descriptor returned", desc.getTableName(), tableAname);
1343      // check HTD attribute
1344      value = desc.getValue(attrName);
1345      assertNotNull("missing HTD attribute value", value);
1346      assertFalse("HTD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
1347      // check HCD attribute
1348      for (ColumnFamilyDescriptor c : desc.getColumnFamilies()) {
1349        value = c.getValue(attrName);
1350        assertNotNull("missing HCD attribute value", value);
1351        assertFalse("HCD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
1352      }
1353    }
1354  }
1355}