001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mob;
019
020import static org.apache.hadoop.hbase.mob.MobConstants.MOB_CLEANER_BATCH_SIZE_UPPER_BOUND;
021import static org.junit.Assert.assertEquals;
022
023import org.apache.hadoop.fs.FileStatus;
024import org.apache.hadoop.fs.Path;
025import org.apache.hadoop.hbase.HBaseClassTestRule;
026import org.apache.hadoop.hbase.HBaseTestingUtil;
027import org.apache.hadoop.hbase.TableName;
028import org.apache.hadoop.hbase.client.Admin;
029import org.apache.hadoop.hbase.client.BufferedMutator;
030import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
031import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
032import org.apache.hadoop.hbase.client.ConnectionFactory;
033import org.apache.hadoop.hbase.client.Put;
034import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
035import org.apache.hadoop.hbase.testclassification.MediumTests;
036import org.apache.hadoop.hbase.util.Bytes;
037import org.apache.hadoop.hbase.util.CommonFSUtils;
038import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
039import org.apache.hadoop.util.ToolRunner;
040import org.junit.After;
041import org.junit.AfterClass;
042import org.junit.Before;
043import org.junit.BeforeClass;
044import org.junit.ClassRule;
045import org.junit.Test;
046import org.junit.experimental.categories.Category;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050@Category(MediumTests.class)
051public class TestExpiredMobFileCleaner {
052
053  @ClassRule
054  public static final HBaseClassTestRule CLASS_RULE =
055    HBaseClassTestRule.forClass(TestExpiredMobFileCleaner.class);
056
057  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
058  private final static TableName tableName = TableName.valueOf("TestExpiredMobFileCleaner");
059  private final static String family = "family";
060  private final static byte[] row1 = Bytes.toBytes("row1");
061  private final static byte[] row2 = Bytes.toBytes("row2");
062  private final static byte[] row3 = Bytes.toBytes("row3");
063  private final static byte[] qf = Bytes.toBytes("qf");
064  private static final Logger LOG = LoggerFactory.getLogger(TestExpiredMobFileCleaner.class);
065
066  private static BufferedMutator table;
067  private static Admin admin;
068
069  @BeforeClass
070  public static void setUpBeforeClass() throws Exception {
071    TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
072    TEST_UTIL.getConfiguration().setInt(MOB_CLEANER_BATCH_SIZE_UPPER_BOUND, 2);
073  }
074
075  @AfterClass
076  public static void tearDownAfterClass() throws Exception {
077
078  }
079
080  @Before
081  public void setUp() throws Exception {
082    TEST_UTIL.startMiniCluster(1);
083  }
084
085  @After
086  public void tearDown() throws Exception {
087    admin.disableTable(tableName);
088    admin.deleteTable(tableName);
089    admin.close();
090    TEST_UTIL.shutdownMiniCluster();
091    TEST_UTIL.getTestFileSystem().delete(TEST_UTIL.getDataTestDir(), true);
092  }
093
094  private void init() throws Exception {
095    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableName);
096    ColumnFamilyDescriptor columnFamilyDescriptor =
097      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).setMobEnabled(true)
098        .setMobThreshold(3L).setMaxVersions(4).build();
099    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
100
101    admin = TEST_UTIL.getAdmin();
102    admin.createTable(tableDescriptorBuilder.build());
103    table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())
104      .getBufferedMutator(tableName);
105  }
106
107  private void modifyColumnExpiryDays(int expireDays) throws Exception {
108    ColumnFamilyDescriptorBuilder columnFamilyDescriptorBuilder = ColumnFamilyDescriptorBuilder
109      .newBuilder(Bytes.toBytes(family)).setMobEnabled(true).setMobThreshold(3L);
110    // change ttl as expire days to make some row expired
111    int timeToLive = expireDays * secondsOfDay();
112    columnFamilyDescriptorBuilder.setTimeToLive(timeToLive);
113
114    admin.modifyColumnFamily(tableName, columnFamilyDescriptorBuilder.build());
115  }
116
117  private void putKVAndFlush(BufferedMutator table, byte[] row, byte[] value, long ts)
118    throws Exception {
119
120    Put put = new Put(row, ts);
121    put.addColumn(Bytes.toBytes(family), qf, value);
122    table.mutate(put);
123
124    table.flush();
125    admin.flush(tableName);
126  }
127
128  /**
129   * Creates a 3 day old hfile and an 1 day old hfile then sets expiry to 2 days. Verifies that the
130   * 3 day old hfile is removed but the 1 day one is still present after the expiry based cleaner is
131   * run.
132   */
133  @Test
134  public void testCleaner() throws Exception {
135    init();
136
137    Path mobDirPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family);
138
139    byte[] dummyData = makeDummyData(600);
140    long ts = EnvironmentEdgeManager.currentTime() - 3 * secondsOfDay() * 1000; // 3 days before
141    putKVAndFlush(table, row1, dummyData, ts);
142    LOG.info("test log to be deleted, tablename is " + tableName);
143    CommonFSUtils.logFileSystemState(TEST_UTIL.getTestFileSystem(),
144      TEST_UTIL.getDefaultRootDirPath(), LOG);
145    FileStatus[] firstFiles = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
146    // the first mob file
147    assertEquals("Before cleanup without delay 1", 1, firstFiles.length);
148    String firstFile = firstFiles[0].getPath().getName();
149
150    // 1.5 day before
151    ts = (long) (EnvironmentEdgeManager.currentTime() - 1.5 * secondsOfDay() * 1000);
152    putKVAndFlush(table, row2, dummyData, ts);
153    FileStatus[] secondFiles = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
154    // now there are 2 mob files
155    assertEquals("Before cleanup without delay 2", 2, secondFiles.length);
156    String f1 = secondFiles[0].getPath().getName();
157    String f2 = secondFiles[1].getPath().getName();
158    String secondFile = f1.equals(firstFile) ? f2 : f1;
159
160    ts = EnvironmentEdgeManager.currentTime() - 4 * secondsOfDay() * 1000; // 4 days before
161    putKVAndFlush(table, row3, dummyData, ts);
162    ts = EnvironmentEdgeManager.currentTime() - 4 * secondsOfDay() * 1000; // 4 days before
163    putKVAndFlush(table, row3, dummyData, ts);
164    FileStatus[] thirdFiles = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
165    // now there are 4 mob files
166    assertEquals("Before cleanup without delay 3", 4, thirdFiles.length);
167
168    modifyColumnExpiryDays(2); // ttl = 2, make the first row expired
169
170    // run the cleaner
171    String[] args = new String[2];
172    args[0] = tableName.getNameAsString();
173    args[1] = family;
174    ToolRunner.run(TEST_UTIL.getConfiguration(), new ExpiredMobFileCleaner(), args);
175
176    FileStatus[] filesAfterClean = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
177    String lastFile = filesAfterClean[0].getPath().getName();
178    // there are 4 mob files in total, but only 3 need to be cleaned
179    assertEquals("After cleanup without delay 1", 1, filesAfterClean.length);
180    assertEquals("After cleanup without delay 2", secondFile, lastFile);
181  }
182
183  private int secondsOfDay() {
184    return 24 * 3600;
185  }
186
187  private byte[] makeDummyData(int size) {
188    byte[] dummyData = new byte[size];
189    Bytes.random(dummyData);
190    return dummyData;
191  }
192}