001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.Set;
025import java.util.concurrent.TimeUnit;
026import java.util.concurrent.atomic.AtomicLong;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.NamespaceDescriptor;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.Admin;
033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
035import org.apache.hadoop.hbase.client.Connection;
036import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
037import org.apache.hadoop.hbase.master.HMaster;
038import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
039import org.apache.hadoop.hbase.testclassification.MediumTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.junit.AfterClass;
042import org.junit.Before;
043import org.junit.BeforeClass;
044import org.junit.ClassRule;
045import org.junit.Rule;
046import org.junit.Test;
047import org.junit.experimental.categories.Category;
048import org.junit.rules.TestName;
049
050/**
051 * Test class for {@link MasterQuotasObserver}.
052 */
053@Category(MediumTests.class)
054public class TestMasterQuotasObserver {
055
056  @ClassRule
057  public static final HBaseClassTestRule CLASS_RULE =
058    HBaseClassTestRule.forClass(TestMasterQuotasObserver.class);
059
060  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
061  private static SpaceQuotaHelperForTests helper;
062
063  @Rule
064  public TestName testName = new TestName();
065
066  @BeforeClass
067  public static void setUp() throws Exception {
068    Configuration conf = TEST_UTIL.getConfiguration();
069    conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
070    TEST_UTIL.startMiniCluster(1);
071  }
072
073  @AfterClass
074  public static void tearDown() throws Exception {
075    TEST_UTIL.shutdownMiniCluster();
076  }
077
078  @Before
079  public void removeAllQuotas() throws Exception {
080    if (helper == null) {
081      helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, new AtomicLong());
082    }
083    final Connection conn = TEST_UTIL.getConnection();
084    // Wait for the quota table to be created
085    if (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)) {
086      helper.waitForQuotaTable(conn);
087    } else {
088      // Or, clean up any quotas from previous test runs.
089      helper.removeAllQuotas(conn);
090      assertEquals(0, helper.listNumDefinedQuotas(conn));
091    }
092  }
093
094  @Test
095  public void testTableSpaceQuotaRemoved() throws Exception {
096    final Connection conn = TEST_UTIL.getConnection();
097    final Admin admin = conn.getAdmin();
098    final TableName tn = TableName.valueOf(testName.getMethodName());
099    // Drop the table if it somehow exists
100    if (admin.tableExists(tn)) {
101      dropTable(admin, tn);
102    }
103    createTable(admin, tn);
104    assertEquals(0, getNumSpaceQuotas());
105
106    // Set space quota
107    QuotaSettings settings =
108      QuotaSettingsFactory.limitTableSpace(tn, 1024L, SpaceViolationPolicy.NO_INSERTS);
109    admin.setQuota(settings);
110    assertEquals(1, getNumSpaceQuotas());
111
112    // Drop the table and observe the Space quota being automatically deleted as well
113    dropTable(admin, tn);
114    assertEquals(0, getNumSpaceQuotas());
115  }
116
117  @Test
118  public void testTableRPCQuotaRemoved() throws Exception {
119    final Connection conn = TEST_UTIL.getConnection();
120    final Admin admin = conn.getAdmin();
121    final TableName tn = TableName.valueOf(testName.getMethodName());
122    // Drop the table if it somehow exists
123    if (admin.tableExists(tn)) {
124      dropTable(admin, tn);
125    }
126
127    createTable(admin, tn);
128    assertEquals(0, getThrottleQuotas());
129
130    // Set RPC quota
131    QuotaSettings settings =
132      QuotaSettingsFactory.throttleTable(tn, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
133    admin.setQuota(settings);
134
135    assertEquals(1, getThrottleQuotas());
136
137    // Delete the table and observe the RPC quota being automatically deleted as well
138    dropTable(admin, tn);
139    assertEquals(0, getThrottleQuotas());
140  }
141
142  @Test
143  public void testTableSpaceAndRPCQuotaRemoved() throws Exception {
144    final Connection conn = TEST_UTIL.getConnection();
145    final Admin admin = conn.getAdmin();
146    final TableName tn = TableName.valueOf(testName.getMethodName());
147    // Drop the table if it somehow exists
148    if (admin.tableExists(tn)) {
149      dropTable(admin, tn);
150    }
151    createTable(admin, tn);
152    assertEquals(0, getNumSpaceQuotas());
153    assertEquals(0, getThrottleQuotas());
154    // Set Both quotas
155    QuotaSettings settings =
156      QuotaSettingsFactory.limitTableSpace(tn, 1024L, SpaceViolationPolicy.NO_INSERTS);
157    admin.setQuota(settings);
158    settings =
159      QuotaSettingsFactory.throttleTable(tn, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
160    admin.setQuota(settings);
161
162    assertEquals(1, getNumSpaceQuotas());
163    assertEquals(1, getThrottleQuotas());
164
165    // Remove Space quota
166    settings = QuotaSettingsFactory.removeTableSpaceLimit(tn);
167    admin.setQuota(settings);
168    assertEquals(0, getNumSpaceQuotas());
169    assertEquals(1, getThrottleQuotas());
170
171    // Set back the space quota
172    settings = QuotaSettingsFactory.limitTableSpace(tn, 1024L, SpaceViolationPolicy.NO_INSERTS);
173    admin.setQuota(settings);
174    assertEquals(1, getNumSpaceQuotas());
175    assertEquals(1, getThrottleQuotas());
176
177    // Remove the throttle quota
178    settings = QuotaSettingsFactory.unthrottleTable(tn);
179    admin.setQuota(settings);
180    assertEquals(1, getNumSpaceQuotas());
181    assertEquals(0, getThrottleQuotas());
182
183    // Set back the throttle quota
184    settings =
185      QuotaSettingsFactory.throttleTable(tn, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
186    admin.setQuota(settings);
187    assertEquals(1, getNumSpaceQuotas());
188    assertEquals(1, getThrottleQuotas());
189
190    // Drop the table and check that both the quotas have been dropped as well
191    dropTable(admin, tn);
192
193    assertEquals(0, getNumSpaceQuotas());
194    assertEquals(0, getThrottleQuotas());
195  }
196
197  @Test
198  public void testNamespaceSpaceQuotaRemoved() throws Exception {
199    final Connection conn = TEST_UTIL.getConnection();
200    final Admin admin = conn.getAdmin();
201    final String ns = testName.getMethodName();
202    // Drop the ns if it somehow exists
203    if (namespaceExists(ns)) {
204      admin.deleteNamespace(ns);
205    }
206
207    // Create the ns
208    NamespaceDescriptor desc = NamespaceDescriptor.create(ns).build();
209    admin.createNamespace(desc);
210    assertEquals(0, getNumSpaceQuotas());
211
212    // Set a quota
213    QuotaSettings settings =
214      QuotaSettingsFactory.limitNamespaceSpace(ns, 1024L, SpaceViolationPolicy.NO_INSERTS);
215    admin.setQuota(settings);
216    assertEquals(1, getNumSpaceQuotas());
217
218    // Delete the namespace and observe the quota being automatically deleted as well
219    admin.deleteNamespace(ns);
220    assertEquals(0, getNumSpaceQuotas());
221  }
222
223  @Test
224  public void testNamespaceRPCQuotaRemoved() throws Exception {
225    final Connection conn = TEST_UTIL.getConnection();
226    final Admin admin = conn.getAdmin();
227    final String ns = testName.getMethodName();
228    // Drop the ns if it somehow exists
229    if (namespaceExists(ns)) {
230      admin.deleteNamespace(ns);
231    }
232
233    // Create the ns
234    NamespaceDescriptor desc = NamespaceDescriptor.create(ns).build();
235    admin.createNamespace(desc);
236    assertEquals(0, getThrottleQuotas());
237
238    // Set a quota
239    QuotaSettings settings =
240      QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
241    admin.setQuota(settings);
242    assertEquals(1, getThrottleQuotas());
243
244    // Delete the namespace and observe the quota being automatically deleted as well
245    admin.deleteNamespace(ns);
246    assertEquals(0, getThrottleQuotas());
247  }
248
249  @Test
250  public void testNamespaceSpaceAndRPCQuotaRemoved() throws Exception {
251    final Connection conn = TEST_UTIL.getConnection();
252    final Admin admin = conn.getAdmin();
253    final String ns = testName.getMethodName();
254    // Drop the ns if it somehow exists
255    if (namespaceExists(ns)) {
256      admin.deleteNamespace(ns);
257    }
258
259    // Create the ns
260    NamespaceDescriptor desc = NamespaceDescriptor.create(ns).build();
261    admin.createNamespace(desc);
262
263    assertEquals(0, getNumSpaceQuotas());
264    assertEquals(0, getThrottleQuotas());
265
266    // Set Both quotas
267    QuotaSettings settings =
268      QuotaSettingsFactory.limitNamespaceSpace(ns, 1024L, SpaceViolationPolicy.NO_INSERTS);
269    admin.setQuota(settings);
270
271    settings =
272      QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
273    admin.setQuota(settings);
274
275    assertEquals(1, getNumSpaceQuotas());
276    assertEquals(1, getThrottleQuotas());
277
278    // Remove Space quota
279    settings = QuotaSettingsFactory.removeNamespaceSpaceLimit(ns);
280    admin.setQuota(settings);
281    assertEquals(0, getNumSpaceQuotas());
282    assertEquals(1, getThrottleQuotas());
283
284    // Set back the space quota
285    settings = QuotaSettingsFactory.limitNamespaceSpace(ns, 1024L, SpaceViolationPolicy.NO_INSERTS);
286    admin.setQuota(settings);
287    assertEquals(1, getNumSpaceQuotas());
288    assertEquals(1, getThrottleQuotas());
289
290    // Remove the throttle quota
291    settings = QuotaSettingsFactory.unthrottleNamespace(ns);
292    admin.setQuota(settings);
293    assertEquals(1, getNumSpaceQuotas());
294    assertEquals(0, getThrottleQuotas());
295
296    // Set back the throttle quota
297    settings =
298      QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
299    admin.setQuota(settings);
300    assertEquals(1, getNumSpaceQuotas());
301    assertEquals(1, getThrottleQuotas());
302
303    // Delete the namespace and check that both the quotas have been dropped as well
304    admin.deleteNamespace(ns);
305
306    assertEquals(0, getNumSpaceQuotas());
307    assertEquals(0, getThrottleQuotas());
308  }
309
310  @Test
311  public void testObserverAddedByDefault() throws Exception {
312    final HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
313    final MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
314    Set<String> coprocessorNames = cpHost.getCoprocessors();
315    assertTrue("Did not find MasterQuotasObserver in list of CPs: " + coprocessorNames,
316      coprocessorNames.contains(MasterQuotasObserver.class.getSimpleName()));
317  }
318
319  public boolean namespaceExists(String ns) throws IOException {
320    NamespaceDescriptor[] descs = TEST_UTIL.getAdmin().listNamespaceDescriptors();
321    for (NamespaceDescriptor desc : descs) {
322      if (ns.equals(desc.getName())) {
323        return true;
324      }
325    }
326    return false;
327  }
328
329  public int getNumSpaceQuotas() throws Exception {
330    try (QuotaRetriever scanner = new QuotaRetriever(TEST_UTIL.getConnection())) {
331      int numSpaceQuotas = 0;
332      for (QuotaSettings quotaSettings : scanner) {
333        if (quotaSettings.getQuotaType() == QuotaType.SPACE) {
334          numSpaceQuotas++;
335        }
336      }
337      return numSpaceQuotas;
338    }
339  }
340
341  public int getThrottleQuotas() throws Exception {
342    try (QuotaRetriever scanner = new QuotaRetriever(TEST_UTIL.getConnection())) {
343      int throttleQuotas = 0;
344      for (QuotaSettings quotaSettings : scanner) {
345        if (quotaSettings.getQuotaType() == QuotaType.THROTTLE) {
346          throttleQuotas++;
347        }
348      }
349      return throttleQuotas;
350    }
351  }
352
353  private void createTable(Admin admin, TableName tn) throws Exception {
354    // Create a table
355    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tn);
356    ColumnFamilyDescriptor columnFamilyDescriptor =
357      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("F1")).build();
358    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
359    admin.createTable(tableDescriptorBuilder.build());
360  }
361
362  private void dropTable(Admin admin, TableName tn) throws Exception {
363    admin.disableTable(tn);
364    admin.deleteTable(tn);
365  }
366}