001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.wal;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNotSame;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertThrows;
025import static org.junit.Assert.assertTrue;
026
027import java.io.IOException;
028import java.util.NavigableSet;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FSDataOutputStream;
031import org.apache.hadoop.fs.FileSystem;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseTestingUtil;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.KeyValueTestUtil;
037import org.apache.hadoop.hbase.ServerName;
038import org.apache.hadoop.hbase.TableName;
039import org.apache.hadoop.hbase.testclassification.RegionServerTests;
040import org.apache.hadoop.hbase.testclassification.SmallTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.hadoop.hbase.util.CommonFSUtils;
043import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
044import org.apache.hadoop.hbase.wal.WALSplitter.PipelineController;
045import org.junit.ClassRule;
046import org.junit.Test;
047import org.junit.experimental.categories.Category;
048
049/**
050 * Simple testing of a few WAL methods.
051 */
052@Category({ RegionServerTests.class, SmallTests.class })
053public class TestWALMethods {
054
055  @ClassRule
056  public static final HBaseClassTestRule CLASS_RULE =
057    HBaseClassTestRule.forClass(TestWALMethods.class);
058
059  private static final byte[] TEST_REGION = Bytes.toBytes("test_region");
060  private static final TableName TEST_TABLE = TableName.valueOf("test_table");
061
062  private final HBaseTestingUtil util = new HBaseTestingUtil();
063
064  @Test
065  public void testServerNameFromWAL() throws Exception {
066    Path walPath = new Path("/hbase/WALs/regionserver-2.example.com,22101,1487767381290",
067      "regionserver-2.example.com%2C22101%2C1487767381290.null0.1487785392316");
068    ServerName name = AbstractFSWALProvider.getServerNameFromWALDirectoryName(walPath);
069    assertEquals(ServerName.valueOf("regionserver-2.example.com", 22101, 1487767381290L), name);
070  }
071
072  @Test
073  public void testServerNameFromTestWAL() throws Exception {
074    Path walPath = new Path(
075      "/user/example/test-data/12ff1404-68c6-4715-a4b9-775e763842bc/WALs/TestWALRecordReader",
076      "TestWALRecordReader.default.1487787939118");
077    ServerName name = AbstractFSWALProvider.getServerNameFromWALDirectoryName(walPath);
078    assertNull(name);
079  }
080
081  /**
082   * Assert that getSplitEditFilesSorted returns files in expected order and that it skips
083   * moved-aside files.
084   */
085  @Test
086  public void testGetSplitEditFilesSorted() throws IOException {
087    FileSystem fs = FileSystem.get(util.getConfiguration());
088    Path regiondir = util.getDataTestDir("regiondir");
089    fs.delete(regiondir, true);
090    fs.mkdirs(regiondir);
091    Path recoverededits = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
092    String first = WALSplitUtil.formatRecoveredEditsFileName(-1);
093    createFile(fs, recoverededits, first);
094    createFile(fs, recoverededits, WALSplitUtil.formatRecoveredEditsFileName(0));
095    createFile(fs, recoverededits, WALSplitUtil.formatRecoveredEditsFileName(1));
096    createFile(fs, recoverededits, WALSplitUtil.formatRecoveredEditsFileName(11));
097    createFile(fs, recoverededits, WALSplitUtil.formatRecoveredEditsFileName(2));
098    createFile(fs, recoverededits, WALSplitUtil.formatRecoveredEditsFileName(50));
099    String last = WALSplitUtil.formatRecoveredEditsFileName(Long.MAX_VALUE);
100    createFile(fs, recoverededits, last);
101    createFile(fs, recoverededits,
102      Long.toString(Long.MAX_VALUE) + "." + EnvironmentEdgeManager.currentTime());
103
104    final Configuration walConf = new Configuration(util.getConfiguration());
105    CommonFSUtils.setRootDir(walConf, regiondir);
106    (new WALFactory(walConf, "dummyLogName")).getWAL(null);
107
108    NavigableSet<Path> files = WALSplitUtil.getSplitEditFilesSorted(fs, regiondir);
109    assertEquals(7, files.size());
110    assertEquals(files.pollFirst().getName(), first);
111    assertEquals(files.pollLast().getName(), last);
112    assertEquals(files.pollFirst().getName(), WALSplitUtil.formatRecoveredEditsFileName(0));
113    assertEquals(files.pollFirst().getName(), WALSplitUtil.formatRecoveredEditsFileName(1));
114    assertEquals(files.pollFirst().getName(), WALSplitUtil.formatRecoveredEditsFileName(2));
115    assertEquals(files.pollFirst().getName(), WALSplitUtil.formatRecoveredEditsFileName(11));
116  }
117
118  private void createFile(final FileSystem fs, final Path testdir, final String name)
119    throws IOException {
120    FSDataOutputStream fdos = fs.create(new Path(testdir, name), true);
121    fdos.close();
122  }
123
124  @Test
125  public void testRegionEntryBuffer() throws Exception {
126    EntryBuffers.RegionEntryBuffer reb =
127      new EntryBuffers.RegionEntryBuffer(TEST_TABLE, TEST_REGION);
128    assertEquals(0, reb.heapSize());
129
130    reb.appendEntry(createTestLogEntry(1));
131    assertTrue(reb.heapSize() > 0);
132  }
133
134  @Test
135  public void testEntrySink() throws Exception {
136    EntryBuffers sink = new EntryBuffers(new PipelineController(), 1 * 1024 * 1024);
137    for (int i = 0; i < 1000; i++) {
138      WAL.Entry entry = createTestLogEntry(i);
139      sink.appendEntry(entry);
140    }
141
142    assertTrue(sink.totalBuffered > 0);
143    long amountInChunk = sink.totalBuffered;
144    // Get a chunk
145    EntryBuffers.RegionEntryBuffer chunk = sink.getChunkToWrite();
146    assertEquals(chunk.heapSize(), amountInChunk);
147
148    // Make sure it got marked that a thread is "working on this"
149    assertTrue(sink.isRegionCurrentlyWriting(TEST_REGION));
150
151    // Insert some more entries
152    for (int i = 0; i < 500; i++) {
153      WAL.Entry entry = createTestLogEntry(i);
154      sink.appendEntry(entry);
155    }
156    // Asking for another chunk shouldn't work since the first one
157    // is still writing
158    assertNull(sink.getChunkToWrite());
159
160    // If we say we're done writing the first chunk, then we should be able
161    // to get the second
162    sink.doneWriting(chunk);
163
164    EntryBuffers.RegionEntryBuffer chunk2 = sink.getChunkToWrite();
165    assertNotNull(chunk2);
166    assertNotSame(chunk, chunk2);
167    long amountInChunk2 = sink.totalBuffered;
168    // The second chunk had fewer rows than the first
169    assertTrue(amountInChunk2 < amountInChunk);
170
171    sink.doneWriting(chunk2);
172    assertEquals(0, sink.totalBuffered);
173  }
174
175  private WAL.Entry createTestLogEntry(int i) {
176    long seq = i;
177    long now = i * 1000;
178
179    WALEdit edit = new WALEdit();
180    edit.add(KeyValueTestUtil.create("row", "fam", "qual", 1234, "val"));
181    WALKeyImpl key =
182      new WALKeyImpl(TEST_REGION, TEST_TABLE, seq, now, HConstants.DEFAULT_CLUSTER_ID);
183    WAL.Entry entry = new WAL.Entry(key, edit);
184    return entry;
185  }
186
187  @Test
188  public void testParseServerNameFromWALName() {
189    assertEquals(ServerName.valueOf("abc,123,123"),
190      AbstractFSWALProvider.parseServerNameFromWALName("abc,123,123.1.12345.meta"));
191    assertEquals(ServerName.valueOf("abc,123,123"),
192      AbstractFSWALProvider.parseServerNameFromWALName("abc,123,123.12345"));
193    assertEquals(ServerName.valueOf("abc,123,123"),
194      AbstractFSWALProvider.parseServerNameFromWALName("abc,123,123"));
195    assertThrows(IllegalArgumentException.class,
196      () -> AbstractFSWALProvider.parseServerNameFromWALName("test,abc,123,123.12345"));
197    assertThrows(IllegalArgumentException.class,
198      () -> AbstractFSWALProvider.parseServerNameFromWALName("abc"));
199  }
200}