001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.storefiletracker;
019
020import static org.hamcrest.CoreMatchers.hasItems;
021import static org.hamcrest.MatcherAssert.assertThat;
022import static org.junit.Assert.assertEquals;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.List;
027import java.util.stream.Collectors;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
034import org.apache.hadoop.hbase.client.Get;
035import org.apache.hadoop.hbase.client.Put;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.client.RegionInfoBuilder;
038import org.apache.hadoop.hbase.client.Result;
039import org.apache.hadoop.hbase.client.TableDescriptor;
040import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
041import org.apache.hadoop.hbase.regionserver.ChunkCreator;
042import org.apache.hadoop.hbase.regionserver.HRegion;
043import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
044import org.apache.hadoop.hbase.testclassification.MediumTests;
045import org.apache.hadoop.hbase.testclassification.RegionServerTests;
046import org.apache.hadoop.hbase.util.Bytes;
047import org.apache.hadoop.hbase.wal.WAL;
048import org.junit.After;
049import org.junit.Before;
050import org.junit.BeforeClass;
051import org.junit.ClassRule;
052import org.junit.Rule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.junit.rules.TestName;
056import org.junit.runner.RunWith;
057import org.junit.runners.Parameterized;
058import org.junit.runners.Parameterized.Parameter;
059import org.junit.runners.Parameterized.Parameters;
060
061import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
062import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
063
064@RunWith(Parameterized.class)
065@Category({ RegionServerTests.class, MediumTests.class })
066public class TestMigrationStoreFileTracker {
067
068  @ClassRule
069  public static final HBaseClassTestRule CLASS_RULE =
070    HBaseClassTestRule.forClass(TestMigrationStoreFileTracker.class);
071
072  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
073
074  private static final byte[] CF = Bytes.toBytes("cf");
075
076  private static final byte[] CQ = Bytes.toBytes("cq");
077
078  private static final TableDescriptor TD =
079    TableDescriptorBuilder.newBuilder(TableName.valueOf("file_based_tracker"))
080      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build();
081
082  private static final RegionInfo RI = RegionInfoBuilder.newBuilder(TD.getTableName()).build();
083
084  @Rule
085  public TestName name = new TestName();
086
087  @Parameter(0)
088  public StoreFileTrackerFactory.Trackers srcImpl;
089
090  @Parameter(1)
091  public StoreFileTrackerFactory.Trackers dstImpl;
092
093  private HRegion region;
094
095  private Path rootDir;
096
097  private WAL wal;
098
099  @Parameters(name = "{index}: src={0}, dst={1}")
100  public static List<Object[]> params() {
101    List<Object[]> params = new ArrayList<>();
102    for (StoreFileTrackerFactory.Trackers src : StoreFileTrackerFactory.Trackers.values()) {
103      for (StoreFileTrackerFactory.Trackers dst : StoreFileTrackerFactory.Trackers.values()) {
104        if (
105          src == StoreFileTrackerFactory.Trackers.MIGRATION
106            || dst == StoreFileTrackerFactory.Trackers.MIGRATION
107        ) {
108          continue;
109        }
110        if (src.equals(dst)) {
111          continue;
112        }
113        params.add(new Object[] { src, dst });
114      }
115    }
116    return params;
117  }
118
119  @BeforeClass
120  public static void setUpBeforeClass() {
121    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
122      MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
123  }
124
125  @Before
126  public void setUp() throws IOException {
127    Configuration conf = UTIL.getConfiguration();
128    conf.set(MigrationStoreFileTracker.SRC_IMPL, srcImpl.name().toLowerCase());
129    conf.set(MigrationStoreFileTracker.DST_IMPL, dstImpl.name().toLowerCase());
130    rootDir = UTIL.getDataTestDir(name.getMethodName().replaceAll("[=:\\[ ]", "_"));
131    wal = HBaseTestingUtility.createWal(conf, rootDir, RI);
132  }
133
134  @After
135  public void tearDown() throws IOException {
136    if (region != null) {
137      region.close();
138    }
139    Closeables.close(wal, true);
140    UTIL.cleanupTestDir();
141  }
142
143  private List<String> getStoreFiles() {
144    return Iterables.getOnlyElement(region.getStores()).getStorefiles().stream()
145      .map(s -> s.getFileInfo().getPath().getName()).collect(Collectors.toList());
146  }
147
148  private HRegion createRegion(Class<? extends StoreFileTrackerBase> trackerImplClass)
149    throws IOException {
150    Configuration conf = new Configuration(UTIL.getConfiguration());
151    conf.setClass(StoreFileTrackerFactory.TRACKER_IMPL, trackerImplClass, StoreFileTracker.class);
152    return HRegion.createHRegion(RI, rootDir, conf, TD, wal, true);
153  }
154
155  private void reopenRegion(Class<? extends StoreFileTrackerBase> trackerImplClass)
156    throws IOException {
157    region.flush(true);
158    List<String> before = getStoreFiles();
159    region.close();
160    Configuration conf = new Configuration(UTIL.getConfiguration());
161    conf.setClass(StoreFileTrackerFactory.TRACKER_IMPL, trackerImplClass, StoreFileTracker.class);
162    region = HRegion.openHRegion(rootDir, RI, TD, wal, conf);
163    List<String> after = getStoreFiles();
164    assertEquals(before.size(), after.size());
165    assertThat(after, hasItems(before.toArray(new String[0])));
166  }
167
168  private void putData(int start, int end) throws IOException {
169    for (int i = start; i < end; i++) {
170      region.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
171      if (i % 30 == 0) {
172        region.flush(true);
173      }
174    }
175  }
176
177  private void verifyData(int start, int end) throws IOException {
178    for (int i = start; i < end; i++) {
179      Result result = region.get(new Get(Bytes.toBytes(i)));
180      assertEquals(i, Bytes.toInt(result.getValue(CF, CQ)));
181    }
182  }
183
184  @Test
185  public void testMigration() throws IOException {
186    region = createRegion(srcImpl.clazz.asSubclass(StoreFileTrackerBase.class));
187    putData(0, 100);
188    verifyData(0, 100);
189    reopenRegion(MigrationStoreFileTracker.class);
190    verifyData(0, 100);
191    region.compact(true);
192    putData(100, 200);
193    reopenRegion(dstImpl.clazz.asSubclass(StoreFileTrackerBase.class));
194    verifyData(0, 200);
195  }
196}