001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.HashMap;
023import java.util.List;
024import java.util.Map;
025import java.util.Map.Entry;
026import java.util.Objects;
027import org.apache.hadoop.hbase.DoNotRetryIOException;
028import org.apache.hadoop.hbase.TableName;
029import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory.QuotaGlobalsSettingsBypass;
030import org.apache.yetus.audience.InterfaceAudience;
031
032import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
033import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
034import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
035import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
036import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
037import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.TimedQuota;
038
039/**
040 * Implementation of {@link GlobalQuotaSettings} to hide the Protobuf messages we use internally.
041 */
042@InterfaceAudience.Private
043public class GlobalQuotaSettingsImpl extends GlobalQuotaSettings {
044
045  private final QuotaProtos.Throttle throttleProto;
046  private final Boolean bypassGlobals;
047  private final QuotaProtos.SpaceQuota spaceProto;
048
049  protected GlobalQuotaSettingsImpl(String username, TableName tableName, String namespace,
050    String regionServer, QuotaProtos.Quotas quotas) {
051    this(username, tableName, namespace, regionServer,
052      (quotas != null && quotas.hasThrottle() ? quotas.getThrottle() : null),
053      (quotas != null && quotas.hasBypassGlobals() ? quotas.getBypassGlobals() : null),
054      (quotas != null && quotas.hasSpace() ? quotas.getSpace() : null));
055  }
056
057  protected GlobalQuotaSettingsImpl(String userName, TableName tableName, String namespace,
058    String regionServer, QuotaProtos.Throttle throttleProto, Boolean bypassGlobals,
059    QuotaProtos.SpaceQuota spaceProto) {
060    super(userName, tableName, namespace, regionServer);
061    this.throttleProto = throttleProto;
062    this.bypassGlobals = bypassGlobals;
063    this.spaceProto = spaceProto;
064  }
065
066  @Override
067  public List<QuotaSettings> getQuotaSettings() {
068    // Very similar to QuotaSettingsFactory
069    List<QuotaSettings> settings = new ArrayList<>();
070    if (throttleProto != null) {
071      settings.addAll(QuotaSettingsFactory.fromThrottle(getUserName(), getTableName(),
072        getNamespace(), getRegionServer(), throttleProto));
073    }
074    if (bypassGlobals != null && bypassGlobals.booleanValue()) {
075      settings.add(new QuotaGlobalsSettingsBypass(getUserName(), getTableName(), getNamespace(),
076        getRegionServer(), true));
077    }
078    if (spaceProto != null) {
079      settings.add(QuotaSettingsFactory.fromSpace(getTableName(), getNamespace(), spaceProto));
080    }
081    return settings;
082  }
083
084  protected QuotaProtos.Throttle getThrottleProto() {
085    return this.throttleProto;
086  }
087
088  protected Boolean getBypassGlobals() {
089    return this.bypassGlobals;
090  }
091
092  protected QuotaProtos.SpaceQuota getSpaceProto() {
093    return this.spaceProto;
094  }
095
096  /**
097   * Constructs a new {@link Quotas} message from {@code this}.
098   */
099  protected Quotas toQuotas() {
100    QuotaProtos.Quotas.Builder builder = QuotaProtos.Quotas.newBuilder();
101    if (getThrottleProto() != null) {
102      builder.setThrottle(getThrottleProto());
103    }
104    if (getBypassGlobals() != null) {
105      builder.setBypassGlobals(getBypassGlobals());
106    }
107    if (getSpaceProto() != null) {
108      builder.setSpace(getSpaceProto());
109    }
110    return builder.build();
111  }
112
113  private boolean hasThrottle(QuotaProtos.ThrottleType quotaType,
114    QuotaProtos.Throttle.Builder throttleBuilder) {
115    boolean hasThrottle = false;
116    switch (quotaType) {
117      case REQUEST_NUMBER:
118        if (throttleBuilder.hasReqNum()) {
119          hasThrottle = true;
120        }
121        break;
122      case REQUEST_SIZE:
123        if (throttleBuilder.hasReqSize()) {
124          hasThrottle = true;
125        }
126        break;
127      case WRITE_NUMBER:
128        if (throttleBuilder.hasWriteNum()) {
129          hasThrottle = true;
130        }
131        break;
132      case WRITE_SIZE:
133        if (throttleBuilder.hasWriteSize()) {
134          hasThrottle = true;
135        }
136        break;
137      case READ_NUMBER:
138        if (throttleBuilder.hasReadNum()) {
139          hasThrottle = true;
140        }
141        break;
142      case READ_SIZE:
143        if (throttleBuilder.hasReadSize()) {
144          hasThrottle = true;
145        }
146        break;
147      case REQUEST_CAPACITY_UNIT:
148        if (throttleBuilder.hasReqCapacityUnit()) {
149          hasThrottle = true;
150        }
151        break;
152      case READ_CAPACITY_UNIT:
153        if (throttleBuilder.hasReadCapacityUnit()) {
154          hasThrottle = true;
155        }
156        break;
157      case WRITE_CAPACITY_UNIT:
158        if (throttleBuilder.hasWriteCapacityUnit()) {
159          hasThrottle = true;
160        }
161        break;
162      case ATOMIC_READ_SIZE:
163        if (throttleBuilder.hasAtomicReadSize()) {
164          hasThrottle = true;
165        }
166        break;
167      case ATOMIC_REQUEST_NUMBER:
168        if (throttleBuilder.hasAtomicReqNum()) {
169          hasThrottle = true;
170        }
171        break;
172      case ATOMIC_WRITE_SIZE:
173        if (throttleBuilder.hasAtomicWriteSize()) {
174          hasThrottle = true;
175        }
176        break;
177      default:
178    }
179    return hasThrottle;
180  }
181
182  @Override
183  protected GlobalQuotaSettingsImpl merge(QuotaSettings other) throws IOException {
184    // Validate the quota subject
185    validateQuotaTarget(other);
186
187    // Propagate the Throttle
188    QuotaProtos.Throttle.Builder throttleBuilder =
189      throttleProto == null ? null : throttleProto.toBuilder();
190
191    if (other instanceof ThrottleSettings) {
192      ThrottleSettings otherThrottle = (ThrottleSettings) other;
193      if (!otherThrottle.proto.hasType() || !otherThrottle.proto.hasTimedQuota()) {
194        // It means it's a remove request
195        // To prevent the "empty" row in QuotaTableUtil.QUOTA_TABLE_NAME
196
197        QuotaProtos.ThrottleRequest otherProto = otherThrottle.proto;
198        if (
199          throttleBuilder != null && !otherThrottle.proto.hasTimedQuota()
200            && otherThrottle.proto.hasType()
201        ) {
202          switch (otherProto.getType()) {
203            case REQUEST_NUMBER:
204              throttleBuilder.clearReqNum();
205              break;
206            case REQUEST_SIZE:
207              throttleBuilder.clearReqSize();
208              break;
209            case WRITE_NUMBER:
210              throttleBuilder.clearWriteNum();
211              break;
212            case WRITE_SIZE:
213              throttleBuilder.clearWriteSize();
214              break;
215            case READ_NUMBER:
216              throttleBuilder.clearReadNum();
217              break;
218            case READ_SIZE:
219              throttleBuilder.clearReadSize();
220              break;
221            case REQUEST_CAPACITY_UNIT:
222              throttleBuilder.clearReqCapacityUnit();
223              break;
224            case READ_CAPACITY_UNIT:
225              throttleBuilder.clearReadCapacityUnit();
226              break;
227            case WRITE_CAPACITY_UNIT:
228              throttleBuilder.clearWriteCapacityUnit();
229              break;
230            case ATOMIC_READ_SIZE:
231              throttleBuilder.clearAtomicReadSize();
232              break;
233            case ATOMIC_REQUEST_NUMBER:
234              throttleBuilder.clearAtomicReqNum();
235              break;
236            case ATOMIC_WRITE_SIZE:
237              throttleBuilder.clearAtomicWriteSize();
238              break;
239            default:
240          }
241          boolean hasThrottle = false;
242          for (QuotaProtos.ThrottleType quotaType : QuotaProtos.ThrottleType.values()) {
243            hasThrottle = hasThrottle(quotaType, throttleBuilder);
244            if (hasThrottle) {
245              break;
246            }
247          }
248          if (!hasThrottle) {
249            throttleBuilder = null;
250          }
251        } else {
252          throttleBuilder = null;
253        }
254
255      } else {
256        QuotaProtos.ThrottleRequest otherProto = otherThrottle.proto;
257        validateTimedQuota(otherProto.getTimedQuota());
258        if (throttleBuilder == null) {
259          throttleBuilder = QuotaProtos.Throttle.newBuilder();
260        }
261        switch (otherProto.getType()) {
262          case REQUEST_NUMBER:
263            throttleBuilder.setReqNum(otherProto.getTimedQuota());
264            break;
265          case REQUEST_SIZE:
266            throttleBuilder.setReqSize(otherProto.getTimedQuota());
267            break;
268          case WRITE_NUMBER:
269            throttleBuilder.setWriteNum(otherProto.getTimedQuota());
270            break;
271          case WRITE_SIZE:
272            throttleBuilder.setWriteSize(otherProto.getTimedQuota());
273            break;
274          case READ_NUMBER:
275            throttleBuilder.setReadNum(otherProto.getTimedQuota());
276            break;
277          case READ_SIZE:
278            throttleBuilder.setReadSize(otherProto.getTimedQuota());
279            break;
280          case REQUEST_CAPACITY_UNIT:
281            throttleBuilder.setReqCapacityUnit(otherProto.getTimedQuota());
282            break;
283          case READ_CAPACITY_UNIT:
284            throttleBuilder.setReadCapacityUnit(otherProto.getTimedQuota());
285            break;
286          case WRITE_CAPACITY_UNIT:
287            throttleBuilder.setWriteCapacityUnit(otherProto.getTimedQuota());
288            break;
289          case ATOMIC_READ_SIZE:
290            throttleBuilder.setAtomicReadSize(otherProto.getTimedQuota());
291            break;
292          case ATOMIC_REQUEST_NUMBER:
293            throttleBuilder.setAtomicReqNum(otherProto.getTimedQuota());
294            break;
295          case ATOMIC_WRITE_SIZE:
296            throttleBuilder.setAtomicWriteSize(otherProto.getTimedQuota());
297            break;
298          default:
299        }
300      }
301    }
302
303    // Propagate the space quota portion
304    QuotaProtos.SpaceQuota.Builder spaceBuilder =
305      (spaceProto == null ? null : spaceProto.toBuilder());
306    if (other instanceof SpaceLimitSettings) {
307      if (spaceBuilder == null) {
308        spaceBuilder = QuotaProtos.SpaceQuota.newBuilder();
309      }
310      SpaceLimitSettings settingsToMerge = (SpaceLimitSettings) other;
311
312      QuotaProtos.SpaceLimitRequest spaceRequest = settingsToMerge.getProto();
313
314      // The message contained the expect SpaceQuota object
315      if (spaceRequest.hasQuota()) {
316        SpaceQuota quotaToMerge = spaceRequest.getQuota();
317        // Validate that the two settings are for the same target.
318        // SpaceQuotas either apply to a table or a namespace (no user spacequota).
319        if (
320          !Objects.equals(getTableName(), settingsToMerge.getTableName())
321            && !Objects.equals(getNamespace(), settingsToMerge.getNamespace())
322        ) {
323          throw new IllegalArgumentException("Cannot merge " + settingsToMerge + " into " + this);
324        }
325
326        if (quotaToMerge.getRemove()) {
327          // It means it's a remove request
328          // Update the builder to propagate the removal
329          spaceBuilder.setRemove(true).clearSoftLimit().clearViolationPolicy();
330        } else {
331          // Add the new settings to the existing settings
332          spaceBuilder.mergeFrom(quotaToMerge);
333        }
334      }
335    }
336
337    boolean removeSpaceBuilder =
338      (spaceBuilder == null) || (spaceBuilder.hasRemove() && spaceBuilder.getRemove());
339
340    Boolean bypassGlobals = this.bypassGlobals;
341    if (other instanceof QuotaGlobalsSettingsBypass) {
342      bypassGlobals = ((QuotaGlobalsSettingsBypass) other).getBypass();
343    }
344
345    if (throttleBuilder == null && removeSpaceBuilder && bypassGlobals == null) {
346      return null;
347    }
348
349    return new GlobalQuotaSettingsImpl(getUserName(), getTableName(), getNamespace(),
350      getRegionServer(), (throttleBuilder == null ? null : throttleBuilder.build()), bypassGlobals,
351      (removeSpaceBuilder ? null : spaceBuilder.build()));
352  }
353
354  private void validateTimedQuota(final TimedQuota timedQuota) throws IOException {
355    if (timedQuota.getSoftLimit() < 1) {
356      throw new DoNotRetryIOException(new UnsupportedOperationException(
357        "The throttle limit must be greater then 0, got " + timedQuota.getSoftLimit()));
358    }
359  }
360
361  @Override
362  public String toString() {
363    StringBuilder builder = new StringBuilder();
364    builder.append("GlobalQuota: ");
365    if (throttleProto != null) {
366      Map<ThrottleType, TimedQuota> throttleQuotas = buildThrottleQuotas(throttleProto);
367      builder.append(" { TYPE => THROTTLE ");
368      for (Entry<ThrottleType, TimedQuota> entry : throttleQuotas.entrySet()) {
369        final ThrottleType type = entry.getKey();
370        final TimedQuota timedQuota = entry.getValue();
371        builder.append("{THROTTLE_TYPE => ").append(type.name()).append(", LIMIT => ");
372        if (timedQuota.hasSoftLimit()) {
373          switch (type) {
374            case REQUEST_NUMBER:
375            case WRITE_NUMBER:
376            case READ_NUMBER:
377            case ATOMIC_REQUEST_NUMBER:
378              builder.append(String.format("%dreq", timedQuota.getSoftLimit()));
379              break;
380            case REQUEST_SIZE:
381            case WRITE_SIZE:
382            case READ_SIZE:
383            case ATOMIC_READ_SIZE:
384            case ATOMIC_WRITE_SIZE:
385              builder.append(sizeToString(timedQuota.getSoftLimit()));
386              break;
387            case REQUEST_CAPACITY_UNIT:
388            case READ_CAPACITY_UNIT:
389            case WRITE_CAPACITY_UNIT:
390              builder.append(String.format("%dCU", timedQuota.getSoftLimit()));
391            default:
392          }
393        } else if (timedQuota.hasShare()) {
394          builder.append(String.format("%.2f%%", timedQuota.getShare()));
395        }
396        builder.append('/');
397        builder.append(timeToString(ProtobufUtil.toTimeUnit(timedQuota.getTimeUnit())));
398        if (timedQuota.hasScope()) {
399          builder.append(", SCOPE => ");
400          builder.append(timedQuota.getScope().toString());
401        }
402      }
403      builder.append("} } ");
404    } else {
405      builder.append(" {} ");
406    }
407    if (bypassGlobals != null) {
408      builder.append(" { GLOBAL_BYPASS => " + bypassGlobals + " } ");
409    }
410    if (spaceProto != null) {
411      builder.append(" { TYPE => SPACE");
412      if (getTableName() != null) {
413        builder.append(", TABLE => ").append(getTableName());
414      }
415      if (getNamespace() != null) {
416        builder.append(", NAMESPACE => ").append(getNamespace());
417      }
418      if (spaceProto.getRemove()) {
419        builder.append(", REMOVE => ").append(spaceProto.getRemove());
420      } else {
421        builder.append(", LIMIT => ").append(sizeToString(spaceProto.getSoftLimit()));
422        builder.append(", VIOLATION_POLICY => ").append(spaceProto.getViolationPolicy());
423      }
424      builder.append(" } ");
425    }
426    return builder.toString();
427  }
428
429  private Map<ThrottleType, TimedQuota> buildThrottleQuotas(Throttle proto) {
430    HashMap<ThrottleType, TimedQuota> quotas = new HashMap<>();
431    if (proto.hasReadNum()) {
432      quotas.put(ThrottleType.READ_NUMBER, proto.getReadNum());
433    }
434    if (proto.hasReadSize()) {
435      quotas.put(ThrottleType.READ_SIZE, proto.getReadSize());
436    }
437    if (proto.hasReqNum()) {
438      quotas.put(ThrottleType.REQUEST_NUMBER, proto.getReqNum());
439    }
440    if (proto.hasReqSize()) {
441      quotas.put(ThrottleType.REQUEST_SIZE, proto.getReqSize());
442    }
443    if (proto.hasWriteNum()) {
444      quotas.put(ThrottleType.WRITE_NUMBER, proto.getWriteNum());
445    }
446    if (proto.hasWriteSize()) {
447      quotas.put(ThrottleType.WRITE_SIZE, proto.getWriteSize());
448    }
449    if (proto.hasReqCapacityUnit()) {
450      quotas.put(ThrottleType.REQUEST_CAPACITY_UNIT, proto.getReqCapacityUnit());
451    }
452    if (proto.hasReadCapacityUnit()) {
453      quotas.put(ThrottleType.READ_CAPACITY_UNIT, proto.getReqCapacityUnit());
454    }
455    if (proto.hasWriteCapacityUnit()) {
456      quotas.put(ThrottleType.WRITE_CAPACITY_UNIT, proto.getWriteCapacityUnit());
457    }
458    return quotas;
459  }
460}