Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.doGets;
021import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.doPuts;
022import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerNamespaceCacheRefresh;
023import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerTableCacheRefresh;
024import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerUserCacheRefresh;
025import static org.junit.Assert.assertEquals;
026import static org.junit.Assert.assertTrue;
027
028import java.util.concurrent.TimeUnit;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.NamespaceDescriptor;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.client.Admin;
034import org.apache.hadoop.hbase.client.Table;
035import org.apache.hadoop.hbase.security.User;
036import org.apache.hadoop.hbase.testclassification.LargeTests;
037import org.apache.hadoop.hbase.testclassification.RegionServerTests;
038import org.apache.hadoop.hbase.util.Bytes;
039import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
040import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
041import org.junit.After;
042import org.junit.AfterClass;
043import org.junit.BeforeClass;
044import org.junit.ClassRule;
045import org.junit.Test;
046import org.junit.experimental.categories.Category;
047
048@Category({ RegionServerTests.class, LargeTests.class })
049public class TestClusterScopeQuotaThrottle {
050
051  @ClassRule
052  public static final HBaseClassTestRule CLASS_RULE =
053    HBaseClassTestRule.forClass(TestClusterScopeQuotaThrottle.class);
054
055  private final static int REFRESH_TIME = 30 * 60000;
056  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
057
058  private final static TableName[] TABLE_NAMES =
059    new TableName[] { TableName.valueOf("TestQuotaAdmin0"), TableName.valueOf("TestQuotaAdmin1"),
060      TableName.valueOf("TestQuotaAdmin2") };
061  private final static byte[] FAMILY = Bytes.toBytes("cf");
062  private final static byte[] QUALIFIER = Bytes.toBytes("q");
063  private final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("1") };
064  private static Table[] tables;
065
066  private final static String NAMESPACE = "TestNs";
067  private final static TableName TABLE_NAME = TableName.valueOf(NAMESPACE, "TestTable");
068  private static Table table;
069
070  @BeforeClass
071  public static void setUpBeforeClass() throws Exception {
072    TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
073    TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, REFRESH_TIME);
074    TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
075    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
076    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
077    TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
078    TEST_UTIL.startMiniCluster(2);
079    TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
080    QuotaCache.TEST_FORCE_REFRESH = true;
081
082    tables = new Table[TABLE_NAMES.length];
083    for (int i = 0; i < TABLE_NAMES.length; ++i) {
084      TEST_UTIL.createTable(TABLE_NAMES[i], FAMILY);
085      TEST_UTIL.waitTableAvailable(TABLE_NAMES[i]);
086      tables[i] = TEST_UTIL.getConnection().getTableBuilder(TABLE_NAMES[i], null)
087        .setOperationTimeout(10000).build();
088    }
089    TEST_UTIL.getAdmin().createNamespace(NamespaceDescriptor.create(NAMESPACE).build());
090    TEST_UTIL.createTable(TABLE_NAME, FAMILY, SPLITS);
091    TEST_UTIL.waitTableAvailable(TABLE_NAME);
092    table = TEST_UTIL.getConnection().getTableBuilder(TABLE_NAME, null).setOperationTimeout(10000)
093      .build();
094  }
095
096  @AfterClass
097  public static void tearDownAfterClass() throws Exception {
098    EnvironmentEdgeManager.reset();
099    for (int i = 0; i < tables.length; ++i) {
100      if (tables[i] != null) {
101        tables[i].close();
102        TEST_UTIL.deleteTable(TABLE_NAMES[i]);
103      }
104    }
105    TEST_UTIL.deleteTable(TABLE_NAME);
106    TEST_UTIL.getAdmin().deleteNamespace(NAMESPACE);
107    TEST_UTIL.shutdownMiniCluster();
108  }
109
110  @After
111  public void tearDown() throws Exception {
112    ThrottleQuotaTestUtil.clearQuotaCache(TEST_UTIL);
113  }
114
115  @Test
116  public void testNamespaceClusterScopeQuota() throws Exception {
117    final Admin admin = TEST_UTIL.getAdmin();
118    final String NAMESPACE = "default";
119
120    // Add 10req/min limit for write request in cluster scope
121    admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.WRITE_NUMBER, 10,
122      TimeUnit.MINUTES, QuotaScope.CLUSTER));
123    // Add 6req/min limit for read request in machine scope
124    admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.READ_NUMBER, 6,
125      TimeUnit.MINUTES, QuotaScope.MACHINE));
126    triggerNamespaceCacheRefresh(TEST_UTIL, false, TABLE_NAMES[0]);
127    // should execute at max 5 write requests and at max 3 read requests
128    assertEquals(5, doPuts(10, FAMILY, QUALIFIER, tables[0]));
129    assertEquals(6, doGets(10, tables[0]));
130    // Remove all the limits
131    admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
132    triggerNamespaceCacheRefresh(TEST_UTIL, true, TABLE_NAMES[0]);
133  }
134
135  @Test
136  public void testTableClusterScopeQuota() throws Exception {
137    final Admin admin = TEST_UTIL.getAdmin();
138    admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAME, ThrottleType.READ_NUMBER, 20,
139      TimeUnit.HOURS, QuotaScope.CLUSTER));
140    triggerTableCacheRefresh(TEST_UTIL, false, TABLE_NAME);
141    for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
142      for (TableName tableName : rst.getRegionServer().getOnlineTables()) {
143        if (tableName.getNameAsString().equals(TABLE_NAME.getNameAsString())) {
144          int rsRegionNum = rst.getRegionServer().getRegions(tableName).size();
145          if (rsRegionNum == 0) {
146            // If rs has 0 region, the machine limiter is 0 (20 * 0 / 2)
147            break;
148          } else if (rsRegionNum == 1) {
149            // If rs has 1 region, the machine limiter is 10 (20 * 1 / 2)
150            // Read rows from 1 region, so can read 10 first time and 0 second time
151            long count = doGets(20, table);
152            assertTrue(count == 0 || count == 10);
153          } else if (rsRegionNum == 2) {
154            // If rs has 2 regions, the machine limiter is 20 (20 * 2 / 2)
155            assertEquals(20, doGets(20, table));
156          }
157          break;
158        }
159      }
160    }
161    admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAME));
162    triggerTableCacheRefresh(TEST_UTIL, true, TABLE_NAME);
163  }
164
165  @Test
166  public void testUserClusterScopeQuota() throws Exception {
167    final Admin admin = TEST_UTIL.getAdmin();
168    final String userName = User.getCurrent().getShortName();
169
170    // Add 6req/min limit for read request in cluster scope
171    admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.READ_NUMBER, 6,
172      TimeUnit.MINUTES, QuotaScope.CLUSTER));
173    // Add 6req/min limit for write request in machine scope
174    admin.setQuota(
175      QuotaSettingsFactory.throttleUser(userName, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
176    triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAMES);
177    // should execute at max 6 read requests and at max 3 write write requests
178    assertEquals(6, doPuts(10, FAMILY, QUALIFIER, tables[0]));
179    assertEquals(3, doGets(10, tables[0]));
180    // Remove all the limits
181    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
182    triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAMES);
183  }
184
185  @Test // Spews the log w/ triggering of scheduler? HBASE-24035
186  public void testUserNamespaceClusterScopeQuota() throws Exception {
187    final Admin admin = TEST_UTIL.getAdmin();
188    final String userName = User.getCurrent().getShortName();
189    final String namespace = TABLE_NAMES[0].getNamespaceAsString();
190
191    // Add 10req/min limit for read request in cluster scope
192    admin.setQuota(QuotaSettingsFactory.throttleUser(userName, namespace, ThrottleType.READ_NUMBER,
193      10, TimeUnit.MINUTES, QuotaScope.CLUSTER));
194    // Add 6req/min limit for write request in machine scope
195    admin.setQuota(QuotaSettingsFactory.throttleUser(userName, namespace, ThrottleType.WRITE_NUMBER,
196      6, TimeUnit.MINUTES));
197    triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAMES[0]);
198    // should execute at max 5 read requests and at max 6 write requests
199    assertEquals(5, doGets(10, tables[0]));
200    assertEquals(6, doPuts(10, FAMILY, QUALIFIER, tables[0]));
201
202    // Remove all the limits
203    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, namespace));
204    triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAMES[0]);
205  }
206
207  @Test
208  public void testUserTableClusterScopeQuota() throws Exception {
209    final Admin admin = TEST_UTIL.getAdmin();
210    final String userName = User.getCurrent().getShortName();
211    admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAME, ThrottleType.READ_NUMBER,
212      20, TimeUnit.HOURS, QuotaScope.CLUSTER));
213    triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAME);
214    for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
215      for (TableName tableName : rst.getRegionServer().getOnlineTables()) {
216        if (tableName.getNameAsString().equals(TABLE_NAME.getNameAsString())) {
217          int rsRegionNum = rst.getRegionServer().getRegions(tableName).size();
218          if (rsRegionNum == 0) {
219            // If rs has 0 region, the machine limiter is 0 (20 * 0 / 2)
220            break;
221          } else if (rsRegionNum == 1) {
222            // If rs has 1 region, the machine limiter is 10 (20 * 1 / 2)
223            // Read rows from 1 region, so can read 10 first time and 0 second time
224            long count = doGets(20, table);
225            assertTrue(count == 0 || count == 10);
226          } else if (rsRegionNum == 2) {
227            // If rs has 2 regions, the machine limiter is 20 (20 * 2 / 2)
228            assertEquals(20, doGets(20, table));
229          }
230          break;
231        }
232      }
233    }
234    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
235    triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAME);
236  }
237}