001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.procedure2.store.region;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.fail;
023
024import java.io.BufferedReader;
025import java.io.ByteArrayInputStream;
026import java.io.ByteArrayOutputStream;
027import java.io.IOException;
028import java.io.InputStreamReader;
029import java.io.PrintStream;
030import java.nio.charset.StandardCharsets;
031import java.util.ArrayList;
032import java.util.List;
033import org.apache.commons.lang3.mutable.MutableLong;
034import org.apache.hadoop.fs.FileSystem;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.hbase.HBaseClassTestRule;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.io.hfile.HFile;
039import org.apache.hadoop.hbase.master.region.MasterRegionFactory;
040import org.apache.hadoop.hbase.testclassification.MasterTests;
041import org.apache.hadoop.hbase.testclassification.SmallTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.apache.hadoop.hbase.util.CommonFSUtils;
044import org.apache.hadoop.util.ToolRunner;
045import org.junit.ClassRule;
046import org.junit.Test;
047import org.junit.experimental.categories.Category;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051@Category({ MasterTests.class, SmallTests.class })
052public class TestHFileProcedurePrettyPrinter extends RegionProcedureStoreTestBase {
053
054  @ClassRule
055  public static final HBaseClassTestRule CLASS_RULE =
056    HBaseClassTestRule.forClass(TestHFileProcedurePrettyPrinter.class);
057
058  private static final Logger LOG = LoggerFactory.getLogger(TestHFileProcedurePrettyPrinter.class);
059
060  private List<String> checkOutput(BufferedReader reader, MutableLong putCount,
061    MutableLong deleteCount, MutableLong markDeletedCount) throws IOException {
062    putCount.setValue(0);
063    deleteCount.setValue(0);
064    markDeletedCount.setValue(0);
065    List<String> fileScanned = new ArrayList<>();
066    for (;;) {
067      String line = reader.readLine();
068      if (line == null) {
069        return fileScanned;
070      }
071      LOG.info(line);
072      if (line.contains("V: mark deleted")) {
073        markDeletedCount.increment();
074      } else if (line.contains("/Put/")) {
075        putCount.increment();
076      } else if (line.contains("/DeleteFamily/")) {
077        deleteCount.increment();
078      } else if (line.startsWith("Scanning -> ")) {
079        fileScanned.add(line.split(" -> ")[1]);
080      } else {
081        fail("Unrecognized output: " + line);
082      }
083    }
084  }
085
086  @Test
087  public void test() throws Exception {
088    HFileProcedurePrettyPrinter printer = new HFileProcedurePrettyPrinter();
089    // -a or -f is required so passing empty args will cause an error and return a non-zero value.
090    assertNotEquals(0, ToolRunner.run(htu.getConfiguration(), printer, new String[0]));
091    List<RegionProcedureStoreTestProcedure> procs = new ArrayList<>();
092    for (int i = 0; i < 10; i++) {
093      RegionProcedureStoreTestProcedure proc = new RegionProcedureStoreTestProcedure();
094      store.insert(proc, null);
095      procs.add(proc);
096    }
097    store.region.flush(true);
098    for (int i = 0; i < 5; i++) {
099      store.delete(procs.get(i).getProcId());
100    }
101    store.region.flush(true);
102    store.cleanup();
103    store.region.flush(true);
104    Path tableDir = CommonFSUtils.getTableDir(
105      new Path(htu.getDataTestDir(), MasterRegionFactory.MASTER_STORE_DIR),
106      MasterRegionFactory.TABLE_NAME);
107    FileSystem fs = tableDir.getFileSystem(htu.getConfiguration());
108    Path regionDir =
109      fs.listStatus(tableDir, p -> RegionInfo.isEncodedRegionName(Bytes.toBytes(p.getName())))[0]
110        .getPath();
111    List<Path> storefiles = HFile.getStoreFiles(fs, regionDir);
112    ByteArrayOutputStream bos = new ByteArrayOutputStream();
113    PrintStream out = new PrintStream(bos);
114    MutableLong putCount = new MutableLong();
115    MutableLong deleteCount = new MutableLong();
116    MutableLong markDeletedCount = new MutableLong();
117    for (Path file : storefiles) {
118      bos.reset();
119      printer = new HFileProcedurePrettyPrinter(out);
120      assertEquals(0,
121        ToolRunner.run(htu.getConfiguration(), printer, new String[] { "-f", file.toString() }));
122      try (BufferedReader reader =
123        new BufferedReader(new InputStreamReader(new ByteArrayInputStream(bos.toByteArray()),
124          StandardCharsets.UTF_8))) {
125        List<String> fileScanned = checkOutput(reader, putCount, deleteCount, markDeletedCount);
126        assertEquals(1, fileScanned.size());
127        assertEquals(file.toString(), fileScanned.get(0));
128        if (putCount.longValue() == 10) {
129          assertEquals(0, deleteCount.longValue());
130          assertEquals(0, markDeletedCount.longValue());
131        } else if (deleteCount.longValue() == 5) {
132          assertEquals(0, putCount.longValue());
133          assertEquals(0, markDeletedCount.longValue());
134        } else if (markDeletedCount.longValue() == 5) {
135          assertEquals(0, putCount.longValue());
136          assertEquals(0, deleteCount.longValue());
137        } else {
138          fail("Should have entered one of the above 3 branches");
139        }
140      }
141    }
142    bos.reset();
143    printer = new HFileProcedurePrettyPrinter(out);
144    assertEquals(0, ToolRunner.run(htu.getConfiguration(), printer, new String[] { "-a" }));
145    try (BufferedReader reader = new BufferedReader(
146      new InputStreamReader(new ByteArrayInputStream(bos.toByteArray()), StandardCharsets.UTF_8))) {
147      List<String> fileScanned = checkOutput(reader, putCount, deleteCount, markDeletedCount);
148      assertEquals(3, fileScanned.size());
149      assertEquals(10, putCount.longValue());
150      assertEquals(5, deleteCount.longValue());
151      assertEquals(5, markDeletedCount.longValue());
152    }
153  }
154}