Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mapreduce;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertThrows;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.ByteArrayOutputStream;
027import java.io.IOException;
028import java.io.PrintStream;
029import java.util.HashMap;
030import java.util.Map;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.SnapshotDescription;
037import org.apache.hadoop.hbase.client.SnapshotType;
038import org.apache.hadoop.hbase.client.Table;
039import org.apache.hadoop.hbase.client.TableDescriptor;
040import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
041import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
042import org.apache.hadoop.hbase.snapshot.SnapshotTTLExpiredException;
043import org.apache.hadoop.hbase.testclassification.LargeTests;
044import org.apache.hadoop.hbase.testclassification.MapReduceTests;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
047import org.apache.hadoop.hbase.util.LauncherSecurityManager;
048import org.junit.AfterClass;
049import org.junit.BeforeClass;
050import org.junit.ClassRule;
051import org.junit.Rule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.junit.rules.TestName;
055
056/**
057 * Basic test for the CopyTable M/R tool
058 */
059@Category({ MapReduceTests.class, LargeTests.class })
060public class TestCopyTable extends CopyTableTestBase {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064    HBaseClassTestRule.forClass(TestCopyTable.class);
065
066  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
067
068  @Rule
069  public TestName name = new TestName();
070
071  @BeforeClass
072  public static void beforeClass() throws Exception {
073    TEST_UTIL.startMiniCluster(3);
074  }
075
076  @AfterClass
077  public static void afterClass() throws Exception {
078    TEST_UTIL.shutdownMiniCluster();
079  }
080
081  @Override
082  protected Table createSourceTable(TableDescriptor desc) throws Exception {
083    return TEST_UTIL.createTable(desc, null);
084  }
085
086  @Override
087  protected Table createTargetTable(TableDescriptor desc) throws Exception {
088    return TEST_UTIL.createTable(desc, null);
089  }
090
091  @Override
092  protected void dropSourceTable(TableName tableName) throws Exception {
093    TEST_UTIL.deleteTable(tableName);
094  }
095
096  @Override
097  protected void dropTargetTable(TableName tableName) throws Exception {
098    TEST_UTIL.deleteTable(tableName);
099  }
100
101  @Override
102  protected String[] getPeerClusterOptions() throws Exception {
103    return new String[0];
104  }
105
106  /**
107   * Simple end-to-end test
108   */
109  @Test
110  public void testCopyTable() throws Exception {
111    doCopyTableTest(TEST_UTIL.getConfiguration(), false);
112  }
113
114  /**
115   * Simple end-to-end test with bulkload.
116   */
117  @Test
118  public void testCopyTableWithBulkload() throws Exception {
119    doCopyTableTest(TEST_UTIL.getConfiguration(), true);
120  }
121
122  /**
123   * Simple end-to-end test on table with MOB
124   */
125  @Test
126  public void testCopyTableWithMob() throws Exception {
127    doCopyTableTestWithMob(TEST_UTIL.getConfiguration(), false);
128  }
129
130  /**
131   * Simple end-to-end test with bulkload on table with MOB.
132   */
133  @Test
134  public void testCopyTableWithBulkloadWithMob() throws Exception {
135    doCopyTableTestWithMob(TEST_UTIL.getConfiguration(), true);
136  }
137
138  @Test
139  public void testStartStopRow() throws Exception {
140    testStartStopRow(TEST_UTIL.getConfiguration());
141  }
142
143  /**
144   * Test copy of table from sourceTable to targetTable all rows from family a
145   */
146  @Test
147  public void testRenameFamily() throws Exception {
148    testRenameFamily(TEST_UTIL.getConfiguration());
149  }
150
151  /**
152   * Test main method of CopyTable.
153   */
154  @Test
155  public void testMainMethod() throws Exception {
156    String[] emptyArgs = { "-h" };
157    PrintStream oldWriter = System.err;
158    ByteArrayOutputStream data = new ByteArrayOutputStream();
159    PrintStream writer = new PrintStream(data);
160    System.setErr(writer);
161    SecurityManager SECURITY_MANAGER = System.getSecurityManager();
162    LauncherSecurityManager newSecurityManager = new LauncherSecurityManager();
163    System.setSecurityManager(newSecurityManager);
164    try {
165      CopyTable.main(emptyArgs);
166      fail("should be exit");
167    } catch (SecurityException e) {
168      assertEquals(1, newSecurityManager.getExitCode());
169    } finally {
170      System.setErr(oldWriter);
171      System.setSecurityManager(SECURITY_MANAGER);
172    }
173    assertTrue(data.toString().contains("rs.class"));
174    // should print usage information
175    assertTrue(data.toString().contains("Usage:"));
176  }
177
178  private Table createTable(TableName tableName, byte[] family, boolean isMob) throws IOException {
179    if (isMob) {
180      ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(family)
181        .setMobEnabled(true).setMobThreshold(1).build();
182      TableDescriptor desc =
183        TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(cfd).build();
184      return TEST_UTIL.createTable(desc, null);
185    } else {
186      return TEST_UTIL.createTable(tableName, family);
187    }
188  }
189
190  private void testCopyTableBySnapshot(String tablePrefix, boolean bulkLoad, boolean isMob)
191    throws Exception {
192    TableName table1 = TableName.valueOf(tablePrefix + 1);
193    TableName table2 = TableName.valueOf(tablePrefix + 2);
194    String snapshot = tablePrefix + "_snapshot";
195    try (Table t1 = createTable(table1, FAMILY_A, isMob);
196      Table t2 = createTable(table2, FAMILY_A, isMob)) {
197      loadData(t1, FAMILY_A, Bytes.toBytes("qualifier"));
198      TEST_UTIL.getAdmin().snapshot(snapshot, table1);
199      boolean success;
200      if (bulkLoad) {
201        success = runCopy(TEST_UTIL.getConfiguration(),
202          new String[] { "--snapshot", "--new.name=" + table2, "--bulkload", snapshot });
203      } else {
204        success = runCopy(TEST_UTIL.getConfiguration(),
205          new String[] { "--snapshot", "--new.name=" + table2, snapshot });
206      }
207      assertTrue(success);
208      verifyRows(t2, FAMILY_A, Bytes.toBytes("qualifier"));
209    } finally {
210      TEST_UTIL.getAdmin().deleteSnapshot(snapshot);
211      TEST_UTIL.deleteTable(table1);
212      TEST_UTIL.deleteTable(table2);
213    }
214  }
215
216  @Test
217  public void testLoadingSnapshotToTable() throws Exception {
218    testCopyTableBySnapshot("testLoadingSnapshotToTable", false, false);
219  }
220
221  @Test
222  public void testLoadingTtlExpiredSnapshotToTable() throws Exception {
223    String tablePrefix = "testLoadingExpiredSnapshotToTable";
224    TableName table1 = TableName.valueOf(tablePrefix + 1);
225    TableName table2 = TableName.valueOf(tablePrefix + 2);
226    Table t1 = createTable(table1, FAMILY_A, false);
227    createTable(table2, FAMILY_A, false);
228    loadData(t1, FAMILY_A, Bytes.toBytes("qualifier"));
229    String snapshot = tablePrefix + "_snapshot";
230    Map<String, Object> properties = new HashMap<>();
231    properties.put("TTL", 10);
232    SnapshotDescription snapshotDescription = new SnapshotDescription(snapshot, table1,
233      SnapshotType.FLUSH, null, EnvironmentEdgeManager.currentTime(), -1, properties);
234    TEST_UTIL.getAdmin().snapshot(snapshotDescription);
235    boolean isExist =
236      TEST_UTIL.getAdmin().listSnapshots().stream().anyMatch(ele -> snapshot.equals(ele.getName()));
237    assertTrue(isExist);
238    int retry = 6;
239    while (
240      !SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(),
241        snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime()) && retry > 0
242    ) {
243      retry--;
244      Thread.sleep(10 * 1000);
245    }
246    boolean isExpiredSnapshot =
247      SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(),
248        snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime());
249    assertTrue(isExpiredSnapshot);
250    String[] args = new String[] { "--snapshot", "--new.name=" + table2, "--bulkload", snapshot };
251    assertThrows(SnapshotTTLExpiredException.class,
252      () -> runCopy(TEST_UTIL.getConfiguration(), args));
253  }
254
255  @Test
256  public void tsetLoadingSnapshotToMobTable() throws Exception {
257    testCopyTableBySnapshot("testLoadingSnapshotToMobTable", false, true);
258  }
259
260  @Test
261  public void testLoadingSnapshotAndBulkLoadToTable() throws Exception {
262    testCopyTableBySnapshot("testLoadingSnapshotAndBulkLoadToTable", true, false);
263  }
264
265  @Test
266  public void testLoadingSnapshotAndBulkLoadToMobTable() throws Exception {
267    testCopyTableBySnapshot("testLoadingSnapshotAndBulkLoadToMobTable", true, true);
268  }
269
270  @Test
271  public void testLoadingSnapshotWithoutSnapshotName() throws Exception {
272    assertFalse(runCopy(TEST_UTIL.getConfiguration(), new String[] { "--snapshot" }));
273  }
274
275  @Test
276  public void testLoadingSnapshotWithoutDestTable() throws Exception {
277    assertFalse(
278      runCopy(TEST_UTIL.getConfiguration(), new String[] { "--snapshot", "sourceSnapshotName" }));
279  }
280
281}