Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Map;
030import java.util.Set;
031import java.util.concurrent.TimeUnit;
032import org.apache.hadoop.hbase.Cell;
033import org.apache.hadoop.hbase.CellScanner;
034import org.apache.hadoop.hbase.HBaseClassTestRule;
035import org.apache.hadoop.hbase.HBaseTestingUtil;
036import org.apache.hadoop.hbase.HConstants;
037import org.apache.hadoop.hbase.NamespaceDescriptor;
038import org.apache.hadoop.hbase.TableName;
039import org.apache.hadoop.hbase.client.Connection;
040import org.apache.hadoop.hbase.client.ConnectionFactory;
041import org.apache.hadoop.hbase.client.Delete;
042import org.apache.hadoop.hbase.client.Put;
043import org.apache.hadoop.hbase.client.Result;
044import org.apache.hadoop.hbase.client.ResultScanner;
045import org.apache.hadoop.hbase.client.Table;
046import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
047import org.apache.hadoop.hbase.testclassification.MasterTests;
048import org.apache.hadoop.hbase.testclassification.MediumTests;
049import org.junit.After;
050import org.junit.AfterClass;
051import org.junit.Before;
052import org.junit.BeforeClass;
053import org.junit.ClassRule;
054import org.junit.Rule;
055import org.junit.Test;
056import org.junit.experimental.categories.Category;
057import org.junit.rules.TestName;
058
059import org.apache.hbase.thirdparty.com.google.common.collect.HashMultimap;
060import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
061import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
062
063import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
064import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
065import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
066import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
067
068/**
069 * Test the quota table helpers (e.g. CRUD operations)
070 */
071@Category({ MasterTests.class, MediumTests.class })
072public class TestQuotaTableUtil {
073
074  @ClassRule
075  public static final HBaseClassTestRule CLASS_RULE =
076    HBaseClassTestRule.forClass(TestQuotaTableUtil.class);
077
078  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
079  private Connection connection;
080  private int tableNameCounter;
081
082  @Rule
083  public TestName testName = new TestName();
084
085  @Rule
086  public TestName name = new TestName();
087
088  @BeforeClass
089  public static void setUpBeforeClass() throws Exception {
090    TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
091    TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, 2000);
092    TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
093    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
094    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
095    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
096    TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
097    TEST_UTIL.startMiniCluster(1);
098    TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
099  }
100
101  @AfterClass
102  public static void tearDownAfterClass() throws Exception {
103    TEST_UTIL.shutdownMiniCluster();
104  }
105
106  @Before
107  public void before() throws IOException {
108    this.connection = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
109    this.tableNameCounter = 0;
110  }
111
112  @After
113  public void after() throws IOException {
114    this.connection.close();
115  }
116
117  @Test
118  public void testDeleteSnapshots() throws Exception {
119    TableName tn = TableName.valueOf(name.getMethodName());
120    try (Table t = connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
121      Quotas quota =
122        Quotas.newBuilder().setSpace(QuotaProtos.SpaceQuota.newBuilder().setSoftLimit(7L)
123          .setViolationPolicy(QuotaProtos.SpaceViolationPolicy.NO_WRITES).build()).build();
124      QuotaUtil.addTableQuota(connection, tn, quota);
125
126      String snapshotName = name.getMethodName() + "_snapshot";
127      t.put(QuotaTableUtil.createPutForSnapshotSize(tn, snapshotName, 3L));
128      t.put(QuotaTableUtil.createPutForSnapshotSize(tn, snapshotName, 5L));
129      assertEquals(1, QuotaTableUtil.getObservedSnapshotSizes(connection).size());
130
131      List<Delete> deletes = QuotaTableUtil.createDeletesForExistingTableSnapshotSizes(connection);
132      assertEquals(1, deletes.size());
133
134      t.delete(deletes);
135      assertEquals(0, QuotaTableUtil.getObservedSnapshotSizes(connection).size());
136
137      String ns = name.getMethodName();
138      t.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 5L));
139      t.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 3L));
140      assertEquals(3L, QuotaTableUtil.getNamespaceSnapshotSize(connection, ns));
141
142      deletes = QuotaTableUtil.createDeletesForExistingNamespaceSnapshotSizes(connection);
143      assertEquals(1, deletes.size());
144
145      t.delete(deletes);
146      assertEquals(0L, QuotaTableUtil.getNamespaceSnapshotSize(connection, ns));
147
148      t.put(QuotaTableUtil.createPutForSnapshotSize(TableName.valueOf("t1"), "s1", 3L));
149      t.put(QuotaTableUtil.createPutForSnapshotSize(TableName.valueOf("t2"), "s2", 3L));
150      t.put(QuotaTableUtil.createPutForSnapshotSize(TableName.valueOf("t3"), "s3", 3L));
151      t.put(QuotaTableUtil.createPutForSnapshotSize(TableName.valueOf("t4"), "s4", 3L));
152      t.put(QuotaTableUtil.createPutForSnapshotSize(TableName.valueOf("t1"), "s5", 3L));
153
154      t.put(QuotaTableUtil.createPutForNamespaceSnapshotSize("ns1", 3L));
155      t.put(QuotaTableUtil.createPutForNamespaceSnapshotSize("ns2", 3L));
156      t.put(QuotaTableUtil.createPutForNamespaceSnapshotSize("ns3", 3L));
157
158      assertEquals(5, QuotaTableUtil.getTableSnapshots(connection).size());
159      assertEquals(3, QuotaTableUtil.getNamespaceSnapshots(connection).size());
160
161      Multimap<TableName, String> tableSnapshotEntriesToRemove = HashMultimap.create();
162      tableSnapshotEntriesToRemove.put(TableName.valueOf("t1"), "s1");
163      tableSnapshotEntriesToRemove.put(TableName.valueOf("t3"), "s3");
164      tableSnapshotEntriesToRemove.put(TableName.valueOf("t4"), "s4");
165
166      Set<String> namespaceSnapshotEntriesToRemove = new HashSet<>();
167      namespaceSnapshotEntriesToRemove.add("ns2");
168      namespaceSnapshotEntriesToRemove.add("ns1");
169
170      deletes =
171        QuotaTableUtil.createDeletesForExistingTableSnapshotSizes(tableSnapshotEntriesToRemove);
172      assertEquals(3, deletes.size());
173      deletes = QuotaTableUtil
174        .createDeletesForExistingNamespaceSnapshotSizes(namespaceSnapshotEntriesToRemove);
175      assertEquals(2, deletes.size());
176    }
177  }
178
179  @Test
180  public void testTableQuotaUtil() throws Exception {
181    final TableName tableName = TableName.valueOf(name.getMethodName());
182
183    Quotas quota = Quotas.newBuilder()
184      .setThrottle(Throttle.newBuilder()
185        .setReqNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE))
186        .setWriteNum(ProtobufUtil.toTimedQuota(600, TimeUnit.SECONDS, QuotaScope.MACHINE))
187        .setReadSize(ProtobufUtil.toTimedQuota(8192, TimeUnit.SECONDS, QuotaScope.MACHINE)).build())
188      .build();
189
190    // Add user quota and verify it
191    QuotaUtil.addTableQuota(this.connection, tableName, quota);
192    Quotas resQuota = QuotaUtil.getTableQuota(this.connection, tableName);
193    assertEquals(quota, resQuota);
194
195    // Remove user quota and verify it
196    QuotaUtil.deleteTableQuota(this.connection, tableName);
197    resQuota = QuotaUtil.getTableQuota(this.connection, tableName);
198    assertEquals(null, resQuota);
199  }
200
201  @Test
202  public void testNamespaceQuotaUtil() throws Exception {
203    final String namespace = "testNamespaceQuotaUtilNS";
204
205    Quotas quota = Quotas.newBuilder()
206      .setThrottle(Throttle.newBuilder()
207        .setReqNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE))
208        .setWriteNum(ProtobufUtil.toTimedQuota(600, TimeUnit.SECONDS, QuotaScope.MACHINE))
209        .setReadSize(ProtobufUtil.toTimedQuota(8192, TimeUnit.SECONDS, QuotaScope.MACHINE)).build())
210      .build();
211
212    // Add user quota and verify it
213    QuotaUtil.addNamespaceQuota(this.connection, namespace, quota);
214    Quotas resQuota = QuotaUtil.getNamespaceQuota(this.connection, namespace);
215    assertEquals(quota, resQuota);
216
217    // Remove user quota and verify it
218    QuotaUtil.deleteNamespaceQuota(this.connection, namespace);
219    resQuota = QuotaUtil.getNamespaceQuota(this.connection, namespace);
220    assertEquals(null, resQuota);
221  }
222
223  @Test
224  public void testUserQuotaUtil() throws Exception {
225    final TableName tableName = TableName.valueOf(name.getMethodName());
226    final String namespace = "testNS";
227    final String user = "testUser";
228
229    Quotas quotaNamespace = Quotas.newBuilder()
230      .setThrottle(Throttle.newBuilder()
231        .setReqNum(ProtobufUtil.toTimedQuota(50000, TimeUnit.SECONDS, QuotaScope.MACHINE)).build())
232      .build();
233    Quotas quotaTable = Quotas.newBuilder().setThrottle(Throttle.newBuilder()
234      .setReqNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE))
235      .setWriteNum(ProtobufUtil.toTimedQuota(600, TimeUnit.SECONDS, QuotaScope.MACHINE))
236      .setReadSize(ProtobufUtil.toTimedQuota(10000, TimeUnit.SECONDS, QuotaScope.MACHINE)).build())
237      .build();
238    Quotas quota = Quotas.newBuilder()
239      .setThrottle(Throttle.newBuilder()
240        .setReqSize(ProtobufUtil.toTimedQuota(8192, TimeUnit.SECONDS, QuotaScope.MACHINE))
241        .setWriteSize(ProtobufUtil.toTimedQuota(4096, TimeUnit.SECONDS, QuotaScope.MACHINE))
242        .setReadNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE)).build())
243      .build();
244
245    // Add user global quota
246    QuotaUtil.addUserQuota(this.connection, user, quota);
247    Quotas resQuota = QuotaUtil.getUserQuota(this.connection, user);
248    assertEquals(quota, resQuota);
249
250    // Add user quota for table
251    QuotaUtil.addUserQuota(this.connection, user, tableName, quotaTable);
252    Quotas resQuotaTable = QuotaUtil.getUserQuota(this.connection, user, tableName);
253    assertEquals(quotaTable, resQuotaTable);
254
255    // Add user quota for namespace
256    QuotaUtil.addUserQuota(this.connection, user, namespace, quotaNamespace);
257    Quotas resQuotaNS = QuotaUtil.getUserQuota(this.connection, user, namespace);
258    assertEquals(quotaNamespace, resQuotaNS);
259
260    // Delete user global quota
261    QuotaUtil.deleteUserQuota(this.connection, user);
262    resQuota = QuotaUtil.getUserQuota(this.connection, user);
263    assertEquals(null, resQuota);
264
265    // Delete user quota for table
266    QuotaUtil.deleteUserQuota(this.connection, user, tableName);
267    resQuotaTable = QuotaUtil.getUserQuota(this.connection, user, tableName);
268    assertEquals(null, resQuotaTable);
269
270    // Delete user quota for namespace
271    QuotaUtil.deleteUserQuota(this.connection, user, namespace);
272    resQuotaNS = QuotaUtil.getUserQuota(this.connection, user, namespace);
273    assertEquals(null, resQuotaNS);
274  }
275
276  @Test
277  public void testSerDeViolationPolicies() throws Exception {
278    final TableName tn1 = getUniqueTableName();
279    final SpaceQuotaSnapshot snapshot1 =
280      new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 512L, 1024L);
281    final TableName tn2 = getUniqueTableName();
282    final SpaceQuotaSnapshot snapshot2 =
283      new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 512L, 1024L);
284    final TableName tn3 = getUniqueTableName();
285    final SpaceQuotaSnapshot snapshot3 =
286      new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 512L, 1024L);
287    List<Put> puts = new ArrayList<>();
288    puts.add(QuotaTableUtil.createPutForSpaceSnapshot(tn1, snapshot1));
289    puts.add(QuotaTableUtil.createPutForSpaceSnapshot(tn2, snapshot2));
290    puts.add(QuotaTableUtil.createPutForSpaceSnapshot(tn3, snapshot3));
291    final Map<TableName, SpaceQuotaSnapshot> expectedPolicies = new HashMap<>();
292    expectedPolicies.put(tn1, snapshot1);
293    expectedPolicies.put(tn2, snapshot2);
294    expectedPolicies.put(tn3, snapshot3);
295
296    final Map<TableName, SpaceQuotaSnapshot> actualPolicies = new HashMap<>();
297    try (Table quotaTable = connection.getTable(QuotaUtil.QUOTA_TABLE_NAME)) {
298      quotaTable.put(puts);
299      ResultScanner scanner = quotaTable.getScanner(QuotaTableUtil.makeQuotaSnapshotScan());
300      for (Result r : scanner) {
301        QuotaTableUtil.extractQuotaSnapshot(r, actualPolicies);
302      }
303      scanner.close();
304    }
305
306    assertEquals(expectedPolicies, actualPolicies);
307  }
308
309  @Test
310  public void testSerdeTableSnapshotSizes() throws Exception {
311    TableName tn1 = TableName.valueOf("tn1");
312    TableName tn2 = TableName.valueOf("tn2");
313    try (Table quotaTable = connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
314      for (int i = 0; i < 3; i++) {
315        Put p = QuotaTableUtil.createPutForSnapshotSize(tn1, "tn1snap" + i, 1024L * (1 + i));
316        quotaTable.put(p);
317      }
318      for (int i = 0; i < 3; i++) {
319        Put p = QuotaTableUtil.createPutForSnapshotSize(tn2, "tn2snap" + i, 2048L * (1 + i));
320        quotaTable.put(p);
321      }
322
323      verifyTableSnapshotSize(quotaTable, tn1, "tn1snap0", 1024L);
324      verifyTableSnapshotSize(quotaTable, tn1, "tn1snap1", 2048L);
325      verifyTableSnapshotSize(quotaTable, tn1, "tn1snap2", 3072L);
326
327      verifyTableSnapshotSize(quotaTable, tn2, "tn2snap0", 2048L);
328      verifyTableSnapshotSize(quotaTable, tn2, "tn2snap1", 4096L);
329      verifyTableSnapshotSize(quotaTable, tn2, "tn2snap2", 6144L);
330
331      cleanUpSnapshotSizes();
332    }
333  }
334
335  @Test
336  public void testReadNamespaceSnapshotSizes() throws Exception {
337    String ns1 = "ns1";
338    String ns2 = "ns2";
339    String defaultNs = NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR;
340    try (Table quotaTable = connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
341      quotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns1, 1024L));
342      quotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns2, 2048L));
343      quotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(defaultNs, 8192L));
344
345      assertEquals(1024L, QuotaTableUtil.getNamespaceSnapshotSize(connection, ns1));
346      assertEquals(2048L, QuotaTableUtil.getNamespaceSnapshotSize(connection, ns2));
347      assertEquals(8192L, QuotaTableUtil.getNamespaceSnapshotSize(connection, defaultNs));
348
349      cleanUpSnapshotSizes();
350    }
351  }
352
353  private TableName getUniqueTableName() {
354    return TableName.valueOf(testName.getMethodName() + "_" + tableNameCounter++);
355  }
356
357  private void verifyTableSnapshotSize(Table quotaTable, TableName tn, String snapshotName,
358    long expectedSize) throws IOException {
359    Result r = quotaTable.get(QuotaTableUtil.makeGetForSnapshotSize(tn, snapshotName));
360    CellScanner cs = r.cellScanner();
361    assertTrue(cs.advance());
362    Cell c = cs.current();
363    assertEquals(expectedSize,
364      QuotaProtos.SpaceQuotaSnapshot.parseFrom(
365        UnsafeByteOperations.unsafeWrap(c.getValueArray(), c.getValueOffset(), c.getValueLength()))
366        .getQuotaUsage());
367    assertFalse(cs.advance());
368  }
369
370  private void cleanUpSnapshotSizes() throws IOException {
371    try (Table t = connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
372      QuotaTableUtil.createDeletesForExistingTableSnapshotSizes(connection);
373      List<Delete> deletes =
374        QuotaTableUtil.createDeletesForExistingNamespaceSnapshotSizes(connection);
375      deletes.addAll(QuotaTableUtil.createDeletesForExistingTableSnapshotSizes(connection));
376      t.delete(deletes);
377    }
378  }
379}