001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025
026import java.io.IOException;
027import java.util.concurrent.TimeUnit;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.JMXListener;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.Waiter;
035import org.apache.hadoop.hbase.client.Admin;
036import org.apache.hadoop.hbase.client.Connection;
037import org.apache.hadoop.hbase.client.ConnectionFactory;
038import org.apache.hadoop.hbase.client.RegionInfo;
039import org.apache.hadoop.hbase.client.RegionLocator;
040import org.apache.hadoop.hbase.client.Table;
041import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
042import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
043import org.apache.hadoop.hbase.master.HMaster;
044import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
045import org.apache.hadoop.hbase.testclassification.MediumTests;
046import org.apache.hadoop.hbase.util.Bytes;
047import org.junit.AfterClass;
048import org.junit.Before;
049import org.junit.BeforeClass;
050import org.junit.ClassRule;
051import org.junit.Test;
052import org.junit.experimental.categories.Category;
053import org.slf4j.Logger;
054import org.slf4j.LoggerFactory;
055
056/**
057 * Verify that the Online config Changes on the HRegionServer side are actually happening. We should
058 * add tests for important configurations which will be changed online.
059 */
060
061@Category({ MediumTests.class })
062public class TestRegionServerOnlineConfigChange {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066    HBaseClassTestRule.forClass(TestRegionServerOnlineConfigChange.class);
067
068  private static final Logger LOG =
069    LoggerFactory.getLogger(TestRegionServerOnlineConfigChange.class.getName());
070  private static final long WAIT_TIMEOUT = TimeUnit.MINUTES.toMillis(2);
071  private static HBaseTestingUtil hbaseTestingUtility = new HBaseTestingUtil();
072  private static Configuration conf = null;
073
074  private static Table t1 = null;
075  private static HRegionServer rs1 = null;
076  private static HMaster hMaster = null;
077  private static byte[] r1name = null;
078  private static Region r1 = null;
079
080  private final static String table1Str = "table1";
081  private final static String columnFamily1Str = "columnFamily1";
082  private final static TableName TABLE1 = TableName.valueOf(table1Str);
083  private final static byte[] COLUMN_FAMILY1 = Bytes.toBytes(columnFamily1Str);
084  private final static long MAX_FILE_SIZE = 20 * 1024 * 1024L;
085
086  @BeforeClass
087  public static void setUpBeforeClass() throws Exception {
088    conf = hbaseTestingUtility.getConfiguration();
089    hbaseTestingUtility.startMiniCluster(2);
090    t1 = hbaseTestingUtility.createTable(
091      TableDescriptorBuilder.newBuilder(TABLE1).setMaxFileSize(MAX_FILE_SIZE).build(),
092      new byte[][] { COLUMN_FAMILY1 }, conf);
093  }
094
095  @AfterClass
096  public static void tearDown() throws Exception {
097    hbaseTestingUtility.shutdownMiniCluster();
098  }
099
100  @Before
101  public void setUp() throws Exception {
102    try (RegionLocator locator = hbaseTestingUtility.getConnection().getRegionLocator(TABLE1)) {
103      RegionInfo firstHRI = locator.getAllRegionLocations().get(0).getRegion();
104      r1name = firstHRI.getRegionName();
105      rs1 = hbaseTestingUtility.getHBaseCluster()
106        .getRegionServer(hbaseTestingUtility.getHBaseCluster().getServerWith(r1name));
107      r1 = rs1.getRegion(r1name);
108      hMaster = hbaseTestingUtility.getHBaseCluster().getMaster();
109    }
110  }
111
112  /**
113   * Check if the number of compaction threads changes online
114   */
115  @Test
116  public void testNumCompactionThreadsOnlineChange() {
117    assertNotNull(rs1.getCompactSplitThread());
118    int newNumSmallThreads = rs1.getCompactSplitThread().getSmallCompactionThreadNum() + 1;
119    int newNumLargeThreads = rs1.getCompactSplitThread().getLargeCompactionThreadNum() + 1;
120
121    conf.setInt("hbase.regionserver.thread.compaction.small", newNumSmallThreads);
122    conf.setInt("hbase.regionserver.thread.compaction.large", newNumLargeThreads);
123    rs1.getConfigurationManager().notifyAllObservers(conf);
124
125    assertEquals(newNumSmallThreads, rs1.getCompactSplitThread().getSmallCompactionThreadNum());
126    assertEquals(newNumLargeThreads, rs1.getCompactSplitThread().getLargeCompactionThreadNum());
127  }
128
129  /**
130   * Test that the configurations in the CompactionConfiguration class change properly.
131   */
132  @Test
133  public void testCompactionConfigurationOnlineChange() throws IOException {
134    String strPrefix = "hbase.hstore.compaction.";
135    Store s = r1.getStore(COLUMN_FAMILY1);
136    if (!(s instanceof HStore)) {
137      LOG.error("Can't test the compaction configuration of HStore class. "
138        + "Got a different implementation other than HStore");
139      return;
140    }
141    HStore hstore = (HStore) s;
142
143    // Set the new compaction ratio to a different value.
144    double newCompactionRatio =
145      hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio() + 0.1;
146    conf.setFloat(strPrefix + "ratio", (float) newCompactionRatio);
147
148    // Notify all the observers, which includes the Store object.
149    rs1.getConfigurationManager().notifyAllObservers(conf);
150
151    // Check if the compaction ratio got updated in the Compaction Configuration
152    assertEquals(newCompactionRatio,
153      hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio(), 0.00001);
154
155    // Check if the off peak compaction ratio gets updated.
156    double newOffPeakCompactionRatio =
157      hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak() + 0.1;
158    conf.setFloat(strPrefix + "ratio.offpeak", (float) newOffPeakCompactionRatio);
159    rs1.getConfigurationManager().notifyAllObservers(conf);
160    assertEquals(newOffPeakCompactionRatio,
161      hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak(), 0.00001);
162
163    // Check if the throttle point gets updated.
164    long newThrottlePoint =
165      hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint() + 10;
166    conf.setLong("hbase.regionserver.thread.compaction.throttle", newThrottlePoint);
167    rs1.getConfigurationManager().notifyAllObservers(conf);
168    assertEquals(newThrottlePoint,
169      hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint());
170
171    // Check if the minFilesToCompact gets updated.
172    int newMinFilesToCompact =
173      hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact() + 1;
174    conf.setLong(strPrefix + "min", newMinFilesToCompact);
175    rs1.getConfigurationManager().notifyAllObservers(conf);
176    assertEquals(newMinFilesToCompact,
177      hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact());
178
179    // Check if the maxFilesToCompact gets updated.
180    int newMaxFilesToCompact =
181      hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxFilesToCompact() + 1;
182    conf.setLong(strPrefix + "max", newMaxFilesToCompact);
183    rs1.getConfigurationManager().notifyAllObservers(conf);
184    assertEquals(newMaxFilesToCompact,
185      hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxFilesToCompact());
186
187    // Check OffPeak hours is updated in an online fashion.
188    conf.setLong(CompactionConfiguration.HBASE_HSTORE_OFFPEAK_START_HOUR, 6);
189    conf.setLong(CompactionConfiguration.HBASE_HSTORE_OFFPEAK_END_HOUR, 7);
190    rs1.getConfigurationManager().notifyAllObservers(conf);
191    assertFalse(hstore.getOffPeakHours().isOffPeakHour(4));
192
193    // Check if the minCompactSize gets updated.
194    long newMinCompactSize =
195      hstore.getStoreEngine().getCompactionPolicy().getConf().getMinCompactSize() + 1;
196    conf.setLong(strPrefix + "min.size", newMinCompactSize);
197    rs1.getConfigurationManager().notifyAllObservers(conf);
198    assertEquals(newMinCompactSize,
199      hstore.getStoreEngine().getCompactionPolicy().getConf().getMinCompactSize());
200
201    // Check if the maxCompactSize gets updated.
202    long newMaxCompactSize =
203      hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxCompactSize() - 1;
204    conf.setLong(strPrefix + "max.size", newMaxCompactSize);
205    rs1.getConfigurationManager().notifyAllObservers(conf);
206    assertEquals(newMaxCompactSize,
207      hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxCompactSize());
208    // Check if the offPeakMaxCompactSize gets updated.
209    long newOffpeakMaxCompactSize =
210      hstore.getStoreEngine().getCompactionPolicy().getConf().getOffPeakMaxCompactSize() - 1;
211    conf.setLong(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY,
212      newOffpeakMaxCompactSize);
213    rs1.getConfigurationManager().notifyAllObservers(conf);
214    assertEquals(newOffpeakMaxCompactSize,
215      hstore.getStoreEngine().getCompactionPolicy().getConf().getOffPeakMaxCompactSize());
216
217    // Check if majorCompactionPeriod gets updated.
218    long newMajorCompactionPeriod =
219      hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionPeriod() + 10;
220    conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, newMajorCompactionPeriod);
221    rs1.getConfigurationManager().notifyAllObservers(conf);
222    assertEquals(newMajorCompactionPeriod,
223      hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionPeriod());
224
225    // Check if majorCompactionJitter gets updated.
226    float newMajorCompactionJitter =
227      hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionJitter() + 0.02F;
228    conf.setFloat("hbase.hregion.majorcompaction.jitter", newMajorCompactionJitter);
229    rs1.getConfigurationManager().notifyAllObservers(conf);
230    assertEquals(newMajorCompactionJitter,
231      hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionJitter(), 0.00001);
232  }
233
234  @Test
235  public void removeClosedRegionFromConfigurationManager() throws Exception {
236    try (Connection connection = ConnectionFactory.createConnection(conf)) {
237      Admin admin = connection.getAdmin();
238      assertTrue("The open region doesn't register as a ConfigurationObserver",
239        rs1.getConfigurationManager().containsObserver(r1));
240      admin.move(r1name);
241      hbaseTestingUtility.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
242        @Override
243        public boolean evaluate() throws Exception {
244          return rs1.getOnlineRegion(r1name) == null;
245        }
246      });
247      assertFalse("The closed region is not removed from ConfigurationManager",
248        rs1.getConfigurationManager().containsObserver(r1));
249      admin.move(r1name, rs1.getServerName());
250      hbaseTestingUtility.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
251        @Override
252        public boolean evaluate() throws Exception {
253          return rs1.getOnlineRegion(r1name) != null;
254        }
255      });
256    }
257  }
258
259  @Test
260  public void testStoreConfigurationOnlineChange() {
261    rs1.getConfigurationManager().notifyAllObservers(conf);
262    long actualMaxFileSize = r1.getStore(COLUMN_FAMILY1).getReadOnlyConfiguration()
263      .getLong(TableDescriptorBuilder.MAX_FILESIZE, -1);
264    assertEquals(MAX_FILE_SIZE, actualMaxFileSize);
265  }
266
267  @Test
268  public void testCoprocessorConfigurationOnlineChange() {
269    assertNull(rs1.getRegionServerCoprocessorHost().findCoprocessor(JMXListener.class.getName()));
270    conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, JMXListener.class.getName());
271    rs1.getConfigurationManager().notifyAllObservers(conf);
272    assertNotNull(
273      rs1.getRegionServerCoprocessorHost().findCoprocessor(JMXListener.class.getName()));
274  }
275
276  @Test
277  public void testCoprocessorConfigurationOnlineChangeOnMaster() {
278    assertNull(hMaster.getMasterCoprocessorHost().findCoprocessor(JMXListener.class.getName()));
279    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, JMXListener.class.getName());
280    assertFalse(hMaster.isInMaintenanceMode());
281    hMaster.getConfigurationManager().notifyAllObservers(conf);
282    assertNotNull(hMaster.getMasterCoprocessorHost().findCoprocessor(JMXListener.class.getName()));
283  }
284
285}