Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import java.io.IOException;
021import java.nio.charset.StandardCharsets;
022import java.util.NavigableMap;
023import org.apache.hadoop.conf.Configuration;
024import org.apache.hadoop.fs.FileSystem;
025import org.apache.hadoop.fs.Path;
026import org.apache.hadoop.hbase.client.Durability;
027import org.apache.hadoop.hbase.client.Get;
028import org.apache.hadoop.hbase.client.Put;
029import org.apache.hadoop.hbase.client.Result;
030import org.apache.hadoop.hbase.client.Table;
031import org.apache.hadoop.hbase.log.HBaseMarkers;
032import org.apache.hadoop.hbase.regionserver.HRegion;
033import org.apache.hadoop.hbase.regionserver.Region;
034import org.apache.hadoop.hbase.regionserver.RegionAsTable;
035import org.apache.hadoop.hbase.util.Bytes;
036import org.apache.hadoop.hbase.util.CommonFSUtils;
037import org.apache.hadoop.hbase.util.FSTableDescriptors;
038import org.apache.hadoop.hdfs.MiniDFSCluster;
039import org.slf4j.Logger;
040import org.slf4j.LoggerFactory;
041
042/**
043 * Abstract HBase test class. Initializes a few things that can come in handly like an
044 * HBaseConfiguration and filesystem.
045 * @deprecated since 2.0.0 and will be removed in 3.0.0. Write junit4 unit tests using
046 *             {@link HBaseTestingUtility}.
047 * @see HBaseTestingUtility
048 * @see <a href="https://issues.apache.org/jira/browse/HBASE-11912">HBASE-11912</a>
049 */
050@Deprecated
051public abstract class HBaseTestCase extends junit.framework.TestCase {
052  private static final Logger LOG = LoggerFactory.getLogger(HBaseTestCase.class);
053
054  protected final static byte[] fam1 = Bytes.toBytes("colfamily11");
055  protected final static byte[] fam2 = Bytes.toBytes("colfamily21");
056  protected final static byte[] fam3 = Bytes.toBytes("colfamily31");
057
058  protected static final byte[][] COLUMNS = { fam1, fam2, fam3 };
059
060  private boolean localfs = false;
061  protected static Path testDir = null;
062  protected FileSystem fs = null;
063  protected HRegion meta = null;
064  protected static final char FIRST_CHAR = 'a';
065  protected static final char LAST_CHAR = 'z';
066  protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
067  protected static final byte[] START_KEY_BYTES = { FIRST_CHAR, FIRST_CHAR, FIRST_CHAR };
068  protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
069  protected static final int MAXVERSIONS = 3;
070
071  protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
072
073  public volatile Configuration conf = testUtil.getConfiguration();
074  public final FSTableDescriptors fsTableDescriptors;
075  {
076    try {
077      fsTableDescriptors = new FSTableDescriptors(conf);
078    } catch (IOException e) {
079      throw new RuntimeException("Failed to init descriptors", e);
080    }
081  }
082
083  /** constructor */
084  public HBaseTestCase() {
085    super();
086  }
087
088  /**
089   *   */
090  public HBaseTestCase(String name) {
091    super(name);
092  }
093
094  /**
095   * Note that this method must be called after the mini hdfs cluster has started or we end up with
096   * a local file system.
097   */
098  @Override
099  protected void setUp() throws Exception {
100    super.setUp();
101    localfs = (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
102
103    if (fs == null) {
104      this.fs = FileSystem.get(conf);
105    }
106    try {
107      if (localfs) {
108        testDir = getUnitTestdir(getName());
109        if (fs.exists(testDir)) {
110          fs.delete(testDir, true);
111        }
112      } else {
113        testDir = CommonFSUtils.getRootDir(conf);
114      }
115    } catch (Exception e) {
116      LOG.error(HBaseMarkers.FATAL, "error during setup", e);
117      throw e;
118    }
119  }
120
121  @Override
122  protected void tearDown() throws Exception {
123    try {
124      if (localfs) {
125        if (this.fs.exists(testDir)) {
126          this.fs.delete(testDir, true);
127        }
128      }
129    } catch (Exception e) {
130      LOG.error(HBaseMarkers.FATAL, "error during tear down", e);
131    }
132    super.tearDown();
133  }
134
135  /**
136   * @see HBaseTestingUtility#getBaseTestDir
137   * @return directory to use for this test
138   */
139  protected Path getUnitTestdir(String testName) {
140    return testUtil.getDataTestDir(testName);
141  }
142
143  /**
144   * You must call close on the returned region and then close on the log file it created. Do
145   * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} to close both the region and the WAL.
146   * @return An {@link HRegion}
147   */
148  public HRegion createNewHRegion(HTableDescriptor desc, byte[] startKey, byte[] endKey)
149    throws IOException {
150    return createNewHRegion(desc, startKey, endKey, this.conf);
151  }
152
153  public HRegion createNewHRegion(HTableDescriptor desc, byte[] startKey, byte[] endKey,
154    Configuration conf) throws IOException {
155    HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
156    return HBaseTestingUtility.createRegionAndWAL(hri, testDir, conf, desc);
157  }
158
159  protected HRegion openClosedRegion(final HRegion closedRegion) throws IOException {
160    return HRegion.openHRegion(closedRegion, null);
161  }
162
163  /**
164   * Create a table of name {@code name} with {@link #COLUMNS} for families.
165   * @param name Name to give table.
166   * @return Column descriptor.
167   */
168  protected HTableDescriptor createTableDescriptor(final String name) {
169    return createTableDescriptor(name, MAXVERSIONS);
170  }
171
172  /**
173   * Create a table of name {@code name} with {@link #COLUMNS} for families.
174   * @param name     Name to give table.
175   * @param versions How many versions to allow per column.
176   * @return Column descriptor.
177   */
178  protected HTableDescriptor createTableDescriptor(final String name, final int versions) {
179    return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS, versions,
180      HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
181  }
182
183  /**
184   * Create a table of name {@code name} with {@link #COLUMNS} for families.
185   * @param name     Name to give table.
186   * @param versions How many versions to allow per column.
187   * @return Column descriptor.
188   */
189  protected HTableDescriptor createTableDescriptor(final String name, final int minVersions,
190    final int versions, final int ttl, KeepDeletedCells keepDeleted) {
191    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
192    for (byte[] cfName : new byte[][] { fam1, fam2, fam3 }) {
193      htd.addFamily(
194        new HColumnDescriptor(cfName).setMinVersions(minVersions).setMaxVersions(versions)
195          .setKeepDeletedCells(keepDeleted).setBlockCacheEnabled(false).setTimeToLive(ttl));
196    }
197    return htd;
198  }
199
200  /**
201   * Add content to region <code>r</code> on the passed column <code>column</code>. Adds data of the
202   * from 'aaa', 'aab', etc where key and value are the same.
203   * @return count of what we added.
204   */
205  public static long addContent(final Region r, final byte[] columnFamily, final byte[] column)
206    throws IOException {
207    byte[] startKey = r.getRegionInfo().getStartKey();
208    byte[] endKey = r.getRegionInfo().getEndKey();
209    byte[] startKeyBytes = startKey;
210    if (startKeyBytes == null || startKeyBytes.length == 0) {
211      startKeyBytes = START_KEY_BYTES;
212    }
213    return addContent(new RegionAsTable(r), Bytes.toString(columnFamily), Bytes.toString(column),
214      startKeyBytes, endKey, -1);
215  }
216
217  public static long addContent(final Region r, final byte[] columnFamily) throws IOException {
218    return addContent(r, columnFamily, null);
219  }
220
221  /**
222   * Add content to region <code>r</code> on the passed column <code>column</code>. Adds data of the
223   * from 'aaa', 'aab', etc where key and value are the same.
224   * @return count of what we added.
225   */
226  public static long addContent(final Table updater, final String columnFamily) throws IOException {
227    return addContent(updater, columnFamily, START_KEY_BYTES, null);
228  }
229
230  public static long addContent(final Table updater, final String family, final String column)
231    throws IOException {
232    return addContent(updater, family, column, START_KEY_BYTES, null);
233  }
234
235  /**
236   * Add content to region <code>r</code> on the passed column <code>column</code>. Adds data of the
237   * from 'aaa', 'aab', etc where key and value are the same.
238   * @return count of what we added.
239   */
240  public static long addContent(final Table updater, final String columnFamily,
241    final byte[] startKeyBytes, final byte[] endKey) throws IOException {
242    return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
243  }
244
245  public static long addContent(final Table updater, final String family, String column,
246    final byte[] startKeyBytes, final byte[] endKey) throws IOException {
247    return addContent(updater, family, column, startKeyBytes, endKey, -1);
248  }
249
250  /**
251   * Add content to region <code>r</code> on the passed column <code>column</code>. Adds data of the
252   * from 'aaa', 'aab', etc where key and value are the same.
253   * @return count of what we added.
254   */
255  public static long addContent(final Table updater, final String columnFamily, final String column,
256    final byte[] startKeyBytes, final byte[] endKey, final long ts) throws IOException {
257    long count = 0;
258    // Add rows of three characters. The first character starts with the
259    // 'a' character and runs up to 'z'. Per first character, we run the
260    // second character over same range. And same for the third so rows
261    // (and values) look like this: 'aaa', 'aab', 'aac', etc.
262    char secondCharStart = (char) startKeyBytes[1];
263    char thirdCharStart = (char) startKeyBytes[2];
264    EXIT: for (char c = (char) startKeyBytes[0]; c <= LAST_CHAR; c++) {
265      for (char d = secondCharStart; d <= LAST_CHAR; d++) {
266        for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
267          byte[] t = new byte[] { (byte) c, (byte) d, (byte) e };
268          if (endKey != null && endKey.length > 0 && Bytes.compareTo(endKey, t) <= 0) {
269            break EXIT;
270          }
271          try {
272            Put put;
273            if (ts != -1) {
274              put = new Put(t, ts);
275            } else {
276              put = new Put(t);
277            }
278            try {
279              StringBuilder sb = new StringBuilder();
280              if (column != null && column.contains(":")) {
281                sb.append(column);
282              } else {
283                if (columnFamily != null) {
284                  sb.append(columnFamily);
285                  if (!columnFamily.endsWith(":")) {
286                    sb.append(":");
287                  }
288                  if (column != null) {
289                    sb.append(column);
290                  }
291                }
292              }
293              byte[][] split = CellUtil.parseColumn(Bytes.toBytes(sb.toString()));
294              if (split.length == 1) {
295                byte[] qualifier = new byte[0];
296                put.addColumn(split[0], qualifier, t);
297              } else {
298                put.addColumn(split[0], split[1], t);
299              }
300              put.setDurability(Durability.SKIP_WAL);
301              updater.put(put);
302              count++;
303            } catch (RuntimeException ex) {
304              ex.printStackTrace();
305              throw ex;
306            } catch (IOException ex) {
307              ex.printStackTrace();
308              throw ex;
309            }
310          } catch (RuntimeException ex) {
311            ex.printStackTrace();
312            throw ex;
313          } catch (IOException ex) {
314            ex.printStackTrace();
315            throw ex;
316          }
317        }
318        // Set start character back to FIRST_CHAR after we've done first loop.
319        thirdCharStart = FIRST_CHAR;
320      }
321      secondCharStart = FIRST_CHAR;
322    }
323    return count;
324  }
325
326  protected void assertResultEquals(final HRegion region, final byte[] row, final byte[] family,
327    final byte[] qualifier, final long timestamp, final byte[] value) throws IOException {
328    Get get = new Get(row);
329    get.setTimestamp(timestamp);
330    Result res = region.get(get);
331    NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map = res.getMap();
332    byte[] res_value = map.get(family).get(qualifier).get(timestamp);
333
334    if (value == null) {
335      assertEquals(
336        Bytes.toString(family) + " " + Bytes.toString(qualifier) + " at timestamp " + timestamp,
337        null, res_value);
338    } else {
339      if (res_value == null) {
340        fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) + " at timestamp " + timestamp
341          + "\" was expected to be \"" + Bytes.toStringBinary(value) + " but was null");
342      }
343      if (res_value != null) {
344        assertEquals(
345          Bytes.toString(family) + " " + Bytes.toString(qualifier) + " at timestamp " + timestamp,
346          value, new String(res_value, StandardCharsets.UTF_8));
347      }
348    }
349  }
350
351  /**
352   * Common method to close down a MiniDFSCluster and the associated file system
353   */
354  public static void shutdownDfs(MiniDFSCluster cluster) {
355    if (cluster != null) {
356      LOG.info("Shutting down Mini DFS ");
357      try {
358        cluster.shutdown();
359      } catch (Exception e) {
360        /// Can get a java.lang.reflect.UndeclaredThrowableException thrown
361        // here because of an InterruptedException. Don't let exceptions in
362        // here be cause of test failure.
363      }
364      try {
365        FileSystem fs = cluster.getFileSystem();
366        if (fs != null) {
367          LOG.info("Shutting down FileSystem");
368          fs.close();
369        }
370        FileSystem.closeAll();
371      } catch (IOException e) {
372        LOG.error("error closing file system", e);
373      }
374    }
375  }
376
377  /**
378   * You must call {@link #closeRootAndMeta()} when done after calling this method. It does cleanup.
379   */
380  protected void createMetaRegion() throws IOException {
381    FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(conf);
382    meta = HBaseTestingUtility.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO, testDir, conf,
383      fsTableDescriptors.get(TableName.META_TABLE_NAME));
384  }
385
386  protected void closeRootAndMeta() throws IOException {
387    HBaseTestingUtility.closeRegionAndWAL(meta);
388  }
389
390  public static void assertByteEquals(byte[] expected, byte[] actual) {
391    if (Bytes.compareTo(expected, actual) != 0) {
392      throw new junit.framework.AssertionFailedError(
393        "expected:<" + Bytes.toString(expected) + "> but was:<" + Bytes.toString(actual) + ">");
394    }
395  }
396
397  public static void assertEquals(byte[] expected, byte[] actual) {
398    if (Bytes.compareTo(expected, actual) != 0) {
399      throw new junit.framework.AssertionFailedError("expected:<" + Bytes.toStringBinary(expected)
400        + "> but was:<" + Bytes.toStringBinary(actual) + ">");
401    }
402  }
403}