Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mapreduce;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.ByteArrayOutputStream;
027import java.io.IOException;
028import java.io.PrintStream;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.Cell;
031import org.apache.hadoop.hbase.CellUtil;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtility;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
036import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
037import org.apache.hadoop.hbase.client.Get;
038import org.apache.hadoop.hbase.client.Put;
039import org.apache.hadoop.hbase.client.Result;
040import org.apache.hadoop.hbase.client.Table;
041import org.apache.hadoop.hbase.client.TableDescriptor;
042import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
043import org.apache.hadoop.hbase.mob.MobTestUtil;
044import org.apache.hadoop.hbase.testclassification.LargeTests;
045import org.apache.hadoop.hbase.testclassification.MapReduceTests;
046import org.apache.hadoop.hbase.util.Bytes;
047import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
048import org.apache.hadoop.hbase.util.LauncherSecurityManager;
049import org.apache.hadoop.util.ToolRunner;
050import org.junit.AfterClass;
051import org.junit.Assert;
052import org.junit.BeforeClass;
053import org.junit.ClassRule;
054import org.junit.Rule;
055import org.junit.Test;
056import org.junit.experimental.categories.Category;
057import org.junit.rules.TestName;
058
059/**
060 * Basic test for the CopyTable M/R tool
061 */
062@Category({ MapReduceTests.class, LargeTests.class })
063public class TestCopyTable {
064
065  @ClassRule
066  public static final HBaseClassTestRule CLASS_RULE =
067    HBaseClassTestRule.forClass(TestCopyTable.class);
068
069  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
070  private static final byte[] ROW1 = Bytes.toBytes("row1");
071  private static final byte[] ROW2 = Bytes.toBytes("row2");
072  private static final String FAMILY_A_STRING = "a";
073  private static final String FAMILY_B_STRING = "b";
074  private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
075  private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
076  private static final byte[] QUALIFIER = Bytes.toBytes("q");
077
078  @Rule
079  public TestName name = new TestName();
080
081  @BeforeClass
082  public static void beforeClass() throws Exception {
083    TEST_UTIL.startMiniCluster(3);
084  }
085
086  @AfterClass
087  public static void afterClass() throws Exception {
088    TEST_UTIL.shutdownMiniCluster();
089  }
090
091  private void doCopyTableTest(boolean bulkload) throws Exception {
092    final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
093    final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
094    final byte[] FAMILY = Bytes.toBytes("family");
095    final byte[] COLUMN1 = Bytes.toBytes("c1");
096
097    try (Table t1 = TEST_UTIL.createTable(tableName1, FAMILY);
098      Table t2 = TEST_UTIL.createTable(tableName2, FAMILY);) {
099      // put rows into the first table
100      loadData(t1, FAMILY, COLUMN1);
101
102      CopyTable copy = new CopyTable();
103      int code;
104      if (bulkload) {
105        code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy,
106          new String[] { "--new.name=" + tableName2.getNameAsString(), "--bulkload",
107            tableName1.getNameAsString() });
108      } else {
109        code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy, new String[] {
110          "--new.name=" + tableName2.getNameAsString(), tableName1.getNameAsString() });
111      }
112      assertEquals("copy job failed", 0, code);
113
114      // verify the data was copied into table 2
115      verifyRows(t2, FAMILY, COLUMN1);
116    } finally {
117      TEST_UTIL.deleteTable(tableName1);
118      TEST_UTIL.deleteTable(tableName2);
119    }
120  }
121
122  private void doCopyTableTestWithMob(boolean bulkload) throws Exception {
123    final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
124    final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
125    final byte[] FAMILY = Bytes.toBytes("mob");
126    final byte[] COLUMN1 = Bytes.toBytes("c1");
127
128    ColumnFamilyDescriptorBuilder cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
129
130    cfd.setMobEnabled(true);
131    cfd.setMobThreshold(5);
132    TableDescriptor desc1 =
133      TableDescriptorBuilder.newBuilder(tableName1).setColumnFamily(cfd.build()).build();
134    TableDescriptor desc2 =
135      TableDescriptorBuilder.newBuilder(tableName2).setColumnFamily(cfd.build()).build();
136
137    try (Table t1 = TEST_UTIL.createTable(desc1, null);
138      Table t2 = TEST_UTIL.createTable(desc2, null);) {
139
140      // put rows into the first table
141      for (int i = 0; i < 10; i++) {
142        Put p = new Put(Bytes.toBytes("row" + i));
143        p.addColumn(FAMILY, COLUMN1, COLUMN1);
144        t1.put(p);
145      }
146
147      CopyTable copy = new CopyTable();
148
149      int code;
150      if (bulkload) {
151        code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy,
152          new String[] { "--new.name=" + tableName2.getNameAsString(), "--bulkload",
153            tableName1.getNameAsString() });
154      } else {
155        code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy, new String[] {
156          "--new.name=" + tableName2.getNameAsString(), tableName1.getNameAsString() });
157      }
158      assertEquals("copy job failed", 0, code);
159
160      // verify the data was copied into table 2
161      for (int i = 0; i < 10; i++) {
162        Get g = new Get(Bytes.toBytes("row" + i));
163        Result r = t2.get(g);
164        assertEquals(1, r.size());
165        assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
166        assertEquals("compare row values between two tables",
167          t1.getDescriptor().getValue("row" + i), t2.getDescriptor().getValue("row" + i));
168      }
169
170      assertEquals("compare count of mob rows after table copy",
171        MobTestUtil.countMobRows(TEST_UTIL, t1), MobTestUtil.countMobRows(TEST_UTIL, t2));
172      assertEquals("compare count of mob row values between two tables",
173        t1.getDescriptor().getValues().size(), t2.getDescriptor().getValues().size());
174      assertTrue("The mob row count is 0 but should be > 0",
175        MobTestUtil.countMobRows(TEST_UTIL, t2) > 0);
176    } finally {
177      TEST_UTIL.deleteTable(tableName1);
178      TEST_UTIL.deleteTable(tableName2);
179    }
180  }
181
182  /**
183   * Simple end-to-end test
184   */
185  @Test
186  public void testCopyTable() throws Exception {
187    doCopyTableTest(false);
188  }
189
190  /**
191   * Simple end-to-end test with bulkload.
192   */
193  @Test
194  public void testCopyTableWithBulkload() throws Exception {
195    doCopyTableTest(true);
196  }
197
198  /**
199   * Simple end-to-end test on table with MOB
200   */
201  @Test
202  public void testCopyTableWithMob() throws Exception {
203    doCopyTableTestWithMob(false);
204  }
205
206  /**
207   * Simple end-to-end test with bulkload on table with MOB.
208   */
209  @Test
210  public void testCopyTableWithBulkloadWithMob() throws Exception {
211    doCopyTableTestWithMob(true);
212  }
213
214  @Test
215  public void testStartStopRow() throws Exception {
216    final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
217    final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
218    final byte[] FAMILY = Bytes.toBytes("family");
219    final byte[] COLUMN1 = Bytes.toBytes("c1");
220    final byte[] ROW0 = Bytes.toBytesBinary("\\x01row0");
221    final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1");
222    final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2");
223
224    Table t1 = TEST_UTIL.createTable(tableName1, FAMILY);
225    Table t2 = TEST_UTIL.createTable(tableName2, FAMILY);
226
227    // put rows into the first table
228    Put p = new Put(ROW0);
229    p.addColumn(FAMILY, COLUMN1, COLUMN1);
230    t1.put(p);
231    p = new Put(ROW1);
232    p.addColumn(FAMILY, COLUMN1, COLUMN1);
233    t1.put(p);
234    p = new Put(ROW2);
235    p.addColumn(FAMILY, COLUMN1, COLUMN1);
236    t1.put(p);
237
238    CopyTable copy = new CopyTable();
239    assertEquals(0,
240      ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy,
241        new String[] { "--new.name=" + tableName2, "--startrow=\\x01row1", "--stoprow=\\x01row2",
242          tableName1.getNameAsString() }));
243
244    // verify the data was copied into table 2
245    // row1 exist, row0, row2 do not exist
246    Get g = new Get(ROW1);
247    Result r = t2.get(g);
248    assertEquals(1, r.size());
249    assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
250
251    g = new Get(ROW0);
252    r = t2.get(g);
253    assertEquals(0, r.size());
254
255    g = new Get(ROW2);
256    r = t2.get(g);
257    assertEquals(0, r.size());
258
259    t1.close();
260    t2.close();
261    TEST_UTIL.deleteTable(tableName1);
262    TEST_UTIL.deleteTable(tableName2);
263  }
264
265  /**
266   * Test copy of table from sourceTable to targetTable all rows from family a
267   */
268  @Test
269  public void testRenameFamily() throws Exception {
270    final TableName sourceTable = TableName.valueOf(name.getMethodName() + "source");
271    final TableName targetTable = TableName.valueOf(name.getMethodName() + "-target");
272
273    byte[][] families = { FAMILY_A, FAMILY_B };
274
275    Table t = TEST_UTIL.createTable(sourceTable, families);
276    Table t2 = TEST_UTIL.createTable(targetTable, families);
277    Put p = new Put(ROW1);
278    p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data11"));
279    p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data12"));
280    p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data13"));
281    t.put(p);
282    p = new Put(ROW2);
283    p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Dat21"));
284    p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data22"));
285    p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data23"));
286    t.put(p);
287
288    long currentTime = EnvironmentEdgeManager.currentTime();
289    String[] args = new String[] { "--new.name=" + targetTable, "--families=a:b", "--all.cells",
290      "--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000),
291      "--versions=1", sourceTable.getNameAsString() };
292    assertNull(t2.get(new Get(ROW1)).getRow());
293
294    assertTrue(runCopy(args));
295
296    assertNotNull(t2.get(new Get(ROW1)).getRow());
297    Result res = t2.get(new Get(ROW1));
298    byte[] b1 = res.getValue(FAMILY_B, QUALIFIER);
299    assertEquals("Data13", new String(b1));
300    assertNotNull(t2.get(new Get(ROW2)).getRow());
301    res = t2.get(new Get(ROW2));
302    b1 = res.getValue(FAMILY_A, QUALIFIER);
303    // Data from the family of B is not copied
304    assertNull(b1);
305
306  }
307
308  /**
309   * Test main method of CopyTable.
310   */
311  @Test
312  public void testMainMethod() throws Exception {
313    String[] emptyArgs = { "-h" };
314    PrintStream oldWriter = System.err;
315    ByteArrayOutputStream data = new ByteArrayOutputStream();
316    PrintStream writer = new PrintStream(data);
317    System.setErr(writer);
318    SecurityManager SECURITY_MANAGER = System.getSecurityManager();
319    LauncherSecurityManager newSecurityManager = new LauncherSecurityManager();
320    System.setSecurityManager(newSecurityManager);
321    try {
322      CopyTable.main(emptyArgs);
323      fail("should be exit");
324    } catch (SecurityException e) {
325      assertEquals(1, newSecurityManager.getExitCode());
326    } finally {
327      System.setErr(oldWriter);
328      System.setSecurityManager(SECURITY_MANAGER);
329    }
330    assertTrue(data.toString().contains("rs.class"));
331    // should print usage information
332    assertTrue(data.toString().contains("Usage:"));
333  }
334
335  private boolean runCopy(String[] args) throws Exception {
336    int status =
337      ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), new CopyTable(), args);
338    return status == 0;
339  }
340
341  private void loadData(Table t, byte[] family, byte[] column) throws IOException {
342    for (int i = 0; i < 10; i++) {
343      byte[] row = Bytes.toBytes("row" + i);
344      Put p = new Put(row);
345      p.addColumn(family, column, row);
346      t.put(p);
347    }
348  }
349
350  private void verifyRows(Table t, byte[] family, byte[] column) throws IOException {
351    for (int i = 0; i < 10; i++) {
352      byte[] row = Bytes.toBytes("row" + i);
353      Get g = new Get(row).addFamily(family);
354      Result r = t.get(g);
355      Assert.assertNotNull(r);
356      Assert.assertEquals(1, r.size());
357      Cell cell = r.rawCells()[0];
358      Assert.assertTrue(CellUtil.matchingQualifier(cell, column));
359      Assert.assertEquals(Bytes.compareTo(cell.getValueArray(), cell.getValueOffset(),
360        cell.getValueLength(), row, 0, row.length), 0);
361    }
362  }
363
364  private Table createTable(TableName tableName, byte[] family, boolean isMob) throws IOException {
365    if (isMob) {
366      ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(family)
367        .setMobEnabled(true).setMobThreshold(1).build();
368      TableDescriptor desc =
369        TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(cfd).build();
370      return TEST_UTIL.createTable(desc, null);
371    } else {
372      return TEST_UTIL.createTable(tableName, family);
373    }
374  }
375
376  private void testCopyTableBySnapshot(String tablePrefix, boolean bulkLoad, boolean isMob)
377    throws Exception {
378    TableName table1 = TableName.valueOf(tablePrefix + 1);
379    TableName table2 = TableName.valueOf(tablePrefix + 2);
380    Table t1 = createTable(table1, FAMILY_A, isMob);
381    Table t2 = createTable(table2, FAMILY_A, isMob);
382    loadData(t1, FAMILY_A, Bytes.toBytes("qualifier"));
383    String snapshot = tablePrefix + "_snapshot";
384    TEST_UTIL.getAdmin().snapshot(snapshot, table1);
385    boolean success;
386    if (bulkLoad) {
387      success =
388        runCopy(new String[] { "--snapshot", "--new.name=" + table2, "--bulkload", snapshot });
389    } else {
390      success = runCopy(new String[] { "--snapshot", "--new.name=" + table2, snapshot });
391    }
392    Assert.assertTrue(success);
393    verifyRows(t2, FAMILY_A, Bytes.toBytes("qualifier"));
394  }
395
396  @Test
397  public void testLoadingSnapshotToTable() throws Exception {
398    testCopyTableBySnapshot("testLoadingSnapshotToTable", false, false);
399  }
400
401  @Test
402  public void tsetLoadingSnapshotToMobTable() throws Exception {
403    testCopyTableBySnapshot("testLoadingSnapshotToMobTable", false, true);
404  }
405
406  @Test
407  public void testLoadingSnapshotAndBulkLoadToTable() throws Exception {
408    testCopyTableBySnapshot("testLoadingSnapshotAndBulkLoadToTable", true, false);
409  }
410
411  @Test
412  public void testLoadingSnapshotAndBulkLoadToMobTable() throws Exception {
413    testCopyTableBySnapshot("testLoadingSnapshotAndBulkLoadToMobTable", true, true);
414  }
415
416  @Test
417  public void testLoadingSnapshotToRemoteCluster() throws Exception {
418    Assert.assertFalse(runCopy(
419      new String[] { "--snapshot", "--peerAdr=hbase://remoteHBase", "sourceSnapshotName" }));
420  }
421
422  @Test
423  public void testLoadingSnapshotWithoutSnapshotName() throws Exception {
424    Assert.assertFalse(runCopy(new String[] { "--snapshot", "--peerAdr=hbase://remoteHBase" }));
425  }
426
427  @Test
428  public void testLoadingSnapshotWithoutDestTable() throws Exception {
429    Assert.assertFalse(runCopy(new String[] { "--snapshot", "sourceSnapshotName" }));
430  }
431}