001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mapreduce;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import org.apache.commons.lang3.ArrayUtils;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.Cell;
029import org.apache.hadoop.hbase.CellUtil;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
032import org.apache.hadoop.hbase.client.Get;
033import org.apache.hadoop.hbase.client.Put;
034import org.apache.hadoop.hbase.client.Result;
035import org.apache.hadoop.hbase.client.Table;
036import org.apache.hadoop.hbase.client.TableDescriptor;
037import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
038import org.apache.hadoop.hbase.mob.MobTestUtil;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
041import org.apache.hadoop.util.ToolRunner;
042import org.junit.Rule;
043import org.junit.rules.TestName;
044
045/**
046 * Base class for testing CopyTable MR tool.
047 */
048public abstract class CopyTableTestBase {
049
050  protected static final byte[] ROW1 = Bytes.toBytes("row1");
051  protected static final byte[] ROW2 = Bytes.toBytes("row2");
052  protected static final String FAMILY_A_STRING = "a";
053  protected static final String FAMILY_B_STRING = "b";
054  protected static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
055  protected static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
056  protected static final byte[] QUALIFIER = Bytes.toBytes("q");
057
058  @Rule
059  public TestName name = new TestName();
060
061  protected abstract Table createSourceTable(TableDescriptor desc) throws Exception;
062
063  protected abstract Table createTargetTable(TableDescriptor desc) throws Exception;
064
065  protected abstract void dropSourceTable(TableName tableName) throws Exception;
066
067  protected abstract void dropTargetTable(TableName tableName) throws Exception;
068
069  protected abstract String[] getPeerClusterOptions() throws Exception;
070
071  protected final void loadData(Table t, byte[] family, byte[] column) throws IOException {
072    for (int i = 0; i < 10; i++) {
073      byte[] row = Bytes.toBytes("row" + i);
074      Put p = new Put(row);
075      p.addColumn(family, column, row);
076      t.put(p);
077    }
078  }
079
080  protected final void verifyRows(Table t, byte[] family, byte[] column) throws IOException {
081    for (int i = 0; i < 10; i++) {
082      byte[] row = Bytes.toBytes("row" + i);
083      Get g = new Get(row).addFamily(family);
084      Result r = t.get(g);
085      assertNotNull(r);
086      assertEquals(1, r.size());
087      Cell cell = r.rawCells()[0];
088      assertTrue(CellUtil.matchingQualifier(cell, column));
089      assertEquals(Bytes.compareTo(cell.getValueArray(), cell.getValueOffset(),
090        cell.getValueLength(), row, 0, row.length), 0);
091    }
092  }
093
094  protected final void doCopyTableTest(Configuration conf, boolean bulkload) throws Exception {
095    TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
096    TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
097    byte[] family = Bytes.toBytes("family");
098    byte[] column = Bytes.toBytes("c1");
099    TableDescriptor desc1 = TableDescriptorBuilder.newBuilder(tableName1)
100      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
101    TableDescriptor desc2 = TableDescriptorBuilder.newBuilder(tableName2)
102      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
103
104    try (Table t1 = createSourceTable(desc1); Table t2 = createTargetTable(desc2)) {
105      // put rows into the first table
106      loadData(t1, family, column);
107
108      String[] peerClusterOptions = getPeerClusterOptions();
109      if (bulkload) {
110        assertTrue(runCopy(conf,
111          ArrayUtils.addAll(peerClusterOptions, "--new.name=" + tableName2.getNameAsString(),
112            "--bulkload", tableName1.getNameAsString())));
113      } else {
114        assertTrue(runCopy(conf, ArrayUtils.addAll(peerClusterOptions,
115          "--new.name=" + tableName2.getNameAsString(), tableName1.getNameAsString())));
116      }
117
118      // verify the data was copied into table 2
119      verifyRows(t2, family, column);
120    } finally {
121      dropSourceTable(tableName1);
122      dropTargetTable(tableName2);
123    }
124  }
125
126  protected final void doCopyTableTestWithMob(Configuration conf, boolean bulkload)
127    throws Exception {
128    TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
129    TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
130    byte[] family = Bytes.toBytes("mob");
131    byte[] column = Bytes.toBytes("c1");
132
133    ColumnFamilyDescriptorBuilder cfd = ColumnFamilyDescriptorBuilder.newBuilder(family);
134
135    cfd.setMobEnabled(true);
136    cfd.setMobThreshold(5);
137    TableDescriptor desc1 =
138      TableDescriptorBuilder.newBuilder(tableName1).setColumnFamily(cfd.build()).build();
139    TableDescriptor desc2 =
140      TableDescriptorBuilder.newBuilder(tableName2).setColumnFamily(cfd.build()).build();
141
142    try (Table t1 = createSourceTable(desc1); Table t2 = createTargetTable(desc2)) {
143      // put rows into the first table
144      for (int i = 0; i < 10; i++) {
145        Put p = new Put(Bytes.toBytes("row" + i));
146        p.addColumn(family, column, column);
147        t1.put(p);
148      }
149
150      String[] peerClusterOptions = getPeerClusterOptions();
151      if (bulkload) {
152        assertTrue(runCopy(conf,
153          ArrayUtils.addAll(peerClusterOptions, "--new.name=" + tableName2.getNameAsString(),
154            "--bulkload", tableName1.getNameAsString())));
155      } else {
156        assertTrue(runCopy(conf, ArrayUtils.addAll(peerClusterOptions,
157          "--new.name=" + tableName2.getNameAsString(), tableName1.getNameAsString())));
158      }
159
160      // verify the data was copied into table 2
161      for (int i = 0; i < 10; i++) {
162        Get g = new Get(Bytes.toBytes("row" + i));
163        Result r = t2.get(g);
164        assertEquals(1, r.size());
165        assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], column));
166        assertEquals("compare row values between two tables",
167          t1.getDescriptor().getValue("row" + i), t2.getDescriptor().getValue("row" + i));
168      }
169
170      assertEquals("compare count of mob rows after table copy", MobTestUtil.countMobRows(t1),
171        MobTestUtil.countMobRows(t2));
172      assertEquals("compare count of mob row values between two tables",
173        t1.getDescriptor().getValues().size(), t2.getDescriptor().getValues().size());
174      assertTrue("The mob row count is 0 but should be > 0", MobTestUtil.countMobRows(t2) > 0);
175    } finally {
176      dropSourceTable(tableName1);
177      dropTargetTable(tableName2);
178    }
179  }
180
181  protected final boolean runCopy(Configuration conf, String[] args) throws Exception {
182    int status = ToolRunner.run(conf, new CopyTable(), args);
183    return status == 0;
184  }
185
186  protected final void testStartStopRow(Configuration conf) throws Exception {
187    final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
188    final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
189    final byte[] family = Bytes.toBytes("family");
190    final byte[] column = Bytes.toBytes("c1");
191    final byte[] row0 = Bytes.toBytesBinary("\\x01row0");
192    final byte[] row1 = Bytes.toBytesBinary("\\x01row1");
193    final byte[] row2 = Bytes.toBytesBinary("\\x01row2");
194    TableDescriptor desc1 = TableDescriptorBuilder.newBuilder(tableName1)
195      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
196    TableDescriptor desc2 = TableDescriptorBuilder.newBuilder(tableName2)
197      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
198    try (Table t1 = createSourceTable(desc1); Table t2 = createTargetTable(desc2)) {
199      // put rows into the first table
200      Put p = new Put(row0);
201      p.addColumn(family, column, column);
202      t1.put(p);
203      p = new Put(row1);
204      p.addColumn(family, column, column);
205      t1.put(p);
206      p = new Put(row2);
207      p.addColumn(family, column, column);
208      t1.put(p);
209
210      String[] peerClusterOptions = getPeerClusterOptions();
211      assertTrue(runCopy(conf, ArrayUtils.addAll(peerClusterOptions, "--new.name=" + tableName2,
212        "--startrow=\\x01row1", "--stoprow=\\x01row2", tableName1.getNameAsString())));
213
214      // verify the data was copied into table 2
215      // row1 exist, row0, row2 do not exist
216      Get g = new Get(row1);
217      Result r = t2.get(g);
218      assertEquals(1, r.size());
219      assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], column));
220
221      g = new Get(row0);
222      r = t2.get(g);
223      assertEquals(0, r.size());
224
225      g = new Get(row2);
226      r = t2.get(g);
227      assertEquals(0, r.size());
228    } finally {
229      dropSourceTable(tableName1);
230      dropTargetTable(tableName2);
231    }
232  }
233
234  protected final void testRenameFamily(Configuration conf) throws Exception {
235    TableName sourceTable = TableName.valueOf(name.getMethodName() + "-source");
236    TableName targetTable = TableName.valueOf(name.getMethodName() + "-target");
237
238    TableDescriptor desc1 = TableDescriptorBuilder.newBuilder(sourceTable)
239      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_A))
240      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_B)).build();
241    TableDescriptor desc2 = TableDescriptorBuilder.newBuilder(targetTable)
242      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_A))
243      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_B)).build();
244
245    try (Table t = createSourceTable(desc1); Table t2 = createTargetTable(desc2)) {
246      Put p = new Put(ROW1);
247      p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data11"));
248      p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data12"));
249      p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data13"));
250      t.put(p);
251      p = new Put(ROW2);
252      p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Dat21"));
253      p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data22"));
254      p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data23"));
255      t.put(p);
256
257      long currentTime = EnvironmentEdgeManager.currentTime();
258      String[] args = ArrayUtils.addAll(getPeerClusterOptions(), "--new.name=" + targetTable,
259        "--families=a:b", "--all.cells", "--starttime=" + (currentTime - 100000),
260        "--endtime=" + (currentTime + 100000), "--versions=1", sourceTable.getNameAsString());
261      assertNull(t2.get(new Get(ROW1)).getRow());
262
263      assertTrue(runCopy(conf, args));
264
265      assertNotNull(t2.get(new Get(ROW1)).getRow());
266      Result res = t2.get(new Get(ROW1));
267      byte[] b1 = res.getValue(FAMILY_B, QUALIFIER);
268      assertEquals("Data13", Bytes.toString(b1));
269      assertNotNull(t2.get(new Get(ROW2)).getRow());
270      res = t2.get(new Get(ROW2));
271      b1 = res.getValue(FAMILY_A, QUALIFIER);
272      // Data from the family of B is not copied
273      assertNull(b1);
274    } finally {
275      dropSourceTable(sourceTable);
276      dropTargetTable(targetTable);
277    }
278  }
279}