001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.quotas; 019 020import java.io.IOException; 021import java.util.ArrayList; 022import java.util.HashMap; 023import java.util.List; 024import java.util.Map; 025import java.util.Map.Entry; 026import java.util.Objects; 027import org.apache.hadoop.hbase.DoNotRetryIOException; 028import org.apache.hadoop.hbase.TableName; 029import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory.QuotaGlobalsSettingsBypass; 030import org.apache.yetus.audience.InterfaceAudience; 031 032import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 033import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; 034import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; 035import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota; 036import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle; 037import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.TimedQuota; 038 039/** 040 * Implementation of {@link GlobalQuotaSettings} to hide the Protobuf messages we use internally. 041 */ 042@InterfaceAudience.Private 043public class GlobalQuotaSettingsImpl extends GlobalQuotaSettings { 044 045 private final QuotaProtos.Throttle throttleProto; 046 private final Boolean bypassGlobals; 047 private final QuotaProtos.SpaceQuota spaceProto; 048 049 protected GlobalQuotaSettingsImpl(String username, TableName tableName, String namespace, 050 String regionServer, QuotaProtos.Quotas quotas) { 051 this(username, tableName, namespace, regionServer, 052 (quotas != null && quotas.hasThrottle() ? quotas.getThrottle() : null), 053 (quotas != null && quotas.hasBypassGlobals() ? quotas.getBypassGlobals() : null), 054 (quotas != null && quotas.hasSpace() ? quotas.getSpace() : null)); 055 } 056 057 protected GlobalQuotaSettingsImpl(String userName, TableName tableName, String namespace, 058 String regionServer, QuotaProtos.Throttle throttleProto, Boolean bypassGlobals, 059 QuotaProtos.SpaceQuota spaceProto) { 060 super(userName, tableName, namespace, regionServer); 061 this.throttleProto = throttleProto; 062 this.bypassGlobals = bypassGlobals; 063 this.spaceProto = spaceProto; 064 } 065 066 @Override 067 public List<QuotaSettings> getQuotaSettings() { 068 // Very similar to QuotaSettingsFactory 069 List<QuotaSettings> settings = new ArrayList<>(); 070 if (throttleProto != null) { 071 settings.addAll(QuotaSettingsFactory.fromThrottle(getUserName(), getTableName(), 072 getNamespace(), getRegionServer(), throttleProto)); 073 } 074 if (bypassGlobals != null && bypassGlobals.booleanValue()) { 075 settings.add(new QuotaGlobalsSettingsBypass(getUserName(), getTableName(), getNamespace(), 076 getRegionServer(), true)); 077 } 078 if (spaceProto != null) { 079 settings.add(QuotaSettingsFactory.fromSpace(getTableName(), getNamespace(), spaceProto)); 080 } 081 return settings; 082 } 083 084 protected QuotaProtos.Throttle getThrottleProto() { 085 return this.throttleProto; 086 } 087 088 protected Boolean getBypassGlobals() { 089 return this.bypassGlobals; 090 } 091 092 protected QuotaProtos.SpaceQuota getSpaceProto() { 093 return this.spaceProto; 094 } 095 096 /** 097 * Constructs a new {@link Quotas} message from {@code this}. 098 */ 099 protected Quotas toQuotas() { 100 QuotaProtos.Quotas.Builder builder = QuotaProtos.Quotas.newBuilder(); 101 if (getThrottleProto() != null) { 102 builder.setThrottle(getThrottleProto()); 103 } 104 if (getBypassGlobals() != null) { 105 builder.setBypassGlobals(getBypassGlobals()); 106 } 107 if (getSpaceProto() != null) { 108 builder.setSpace(getSpaceProto()); 109 } 110 return builder.build(); 111 } 112 113 private boolean hasThrottle(QuotaProtos.ThrottleType quotaType, 114 QuotaProtos.Throttle.Builder throttleBuilder) { 115 boolean hasThrottle = false; 116 switch (quotaType) { 117 case REQUEST_NUMBER: 118 if (throttleBuilder.hasReqNum()) { 119 hasThrottle = true; 120 } 121 break; 122 case REQUEST_SIZE: 123 if (throttleBuilder.hasReqSize()) { 124 hasThrottle = true; 125 } 126 break; 127 case WRITE_NUMBER: 128 if (throttleBuilder.hasWriteNum()) { 129 hasThrottle = true; 130 } 131 break; 132 case WRITE_SIZE: 133 if (throttleBuilder.hasWriteSize()) { 134 hasThrottle = true; 135 } 136 break; 137 case READ_NUMBER: 138 if (throttleBuilder.hasReadNum()) { 139 hasThrottle = true; 140 } 141 break; 142 case READ_SIZE: 143 if (throttleBuilder.hasReadSize()) { 144 hasThrottle = true; 145 } 146 break; 147 case REQUEST_CAPACITY_UNIT: 148 if (throttleBuilder.hasReqCapacityUnit()) { 149 hasThrottle = true; 150 } 151 break; 152 case READ_CAPACITY_UNIT: 153 if (throttleBuilder.hasReadCapacityUnit()) { 154 hasThrottle = true; 155 } 156 break; 157 case WRITE_CAPACITY_UNIT: 158 if (throttleBuilder.hasWriteCapacityUnit()) { 159 hasThrottle = true; 160 } 161 break; 162 case ATOMIC_READ_SIZE: 163 if (throttleBuilder.hasAtomicReadSize()) { 164 hasThrottle = true; 165 } 166 break; 167 case ATOMIC_REQUEST_NUMBER: 168 if (throttleBuilder.hasAtomicReqNum()) { 169 hasThrottle = true; 170 } 171 break; 172 case ATOMIC_WRITE_SIZE: 173 if (throttleBuilder.hasAtomicWriteSize()) { 174 hasThrottle = true; 175 } 176 break; 177 default: 178 } 179 return hasThrottle; 180 } 181 182 @Override 183 protected GlobalQuotaSettingsImpl merge(QuotaSettings other) throws IOException { 184 // Validate the quota subject 185 validateQuotaTarget(other); 186 187 // Propagate the Throttle 188 QuotaProtos.Throttle.Builder throttleBuilder = 189 throttleProto == null ? null : throttleProto.toBuilder(); 190 191 if (other instanceof ThrottleSettings) { 192 ThrottleSettings otherThrottle = (ThrottleSettings) other; 193 if (!otherThrottle.proto.hasType() || !otherThrottle.proto.hasTimedQuota()) { 194 // It means it's a remove request 195 // To prevent the "empty" row in QuotaTableUtil.QUOTA_TABLE_NAME 196 197 QuotaProtos.ThrottleRequest otherProto = otherThrottle.proto; 198 if ( 199 throttleBuilder != null && !otherThrottle.proto.hasTimedQuota() 200 && otherThrottle.proto.hasType() 201 ) { 202 switch (otherProto.getType()) { 203 case REQUEST_NUMBER: 204 throttleBuilder.clearReqNum(); 205 break; 206 case REQUEST_SIZE: 207 throttleBuilder.clearReqSize(); 208 break; 209 case WRITE_NUMBER: 210 throttleBuilder.clearWriteNum(); 211 break; 212 case WRITE_SIZE: 213 throttleBuilder.clearWriteSize(); 214 break; 215 case READ_NUMBER: 216 throttleBuilder.clearReadNum(); 217 break; 218 case READ_SIZE: 219 throttleBuilder.clearReadSize(); 220 break; 221 case REQUEST_CAPACITY_UNIT: 222 throttleBuilder.clearReqCapacityUnit(); 223 break; 224 case READ_CAPACITY_UNIT: 225 throttleBuilder.clearReadCapacityUnit(); 226 break; 227 case WRITE_CAPACITY_UNIT: 228 throttleBuilder.clearWriteCapacityUnit(); 229 break; 230 case ATOMIC_READ_SIZE: 231 throttleBuilder.clearAtomicReadSize(); 232 break; 233 case ATOMIC_REQUEST_NUMBER: 234 throttleBuilder.clearAtomicReqNum(); 235 break; 236 case ATOMIC_WRITE_SIZE: 237 throttleBuilder.clearAtomicWriteSize(); 238 break; 239 default: 240 } 241 boolean hasThrottle = false; 242 for (QuotaProtos.ThrottleType quotaType : QuotaProtos.ThrottleType.values()) { 243 hasThrottle = hasThrottle(quotaType, throttleBuilder); 244 if (hasThrottle) { 245 break; 246 } 247 } 248 if (!hasThrottle) { 249 throttleBuilder = null; 250 } 251 } else { 252 throttleBuilder = null; 253 } 254 255 } else { 256 QuotaProtos.ThrottleRequest otherProto = otherThrottle.proto; 257 validateTimedQuota(otherProto.getTimedQuota()); 258 if (throttleBuilder == null) { 259 throttleBuilder = QuotaProtos.Throttle.newBuilder(); 260 } 261 switch (otherProto.getType()) { 262 case REQUEST_NUMBER: 263 throttleBuilder.setReqNum(otherProto.getTimedQuota()); 264 break; 265 case REQUEST_SIZE: 266 throttleBuilder.setReqSize(otherProto.getTimedQuota()); 267 break; 268 case WRITE_NUMBER: 269 throttleBuilder.setWriteNum(otherProto.getTimedQuota()); 270 break; 271 case WRITE_SIZE: 272 throttleBuilder.setWriteSize(otherProto.getTimedQuota()); 273 break; 274 case READ_NUMBER: 275 throttleBuilder.setReadNum(otherProto.getTimedQuota()); 276 break; 277 case READ_SIZE: 278 throttleBuilder.setReadSize(otherProto.getTimedQuota()); 279 break; 280 case REQUEST_CAPACITY_UNIT: 281 throttleBuilder.setReqCapacityUnit(otherProto.getTimedQuota()); 282 break; 283 case READ_CAPACITY_UNIT: 284 throttleBuilder.setReadCapacityUnit(otherProto.getTimedQuota()); 285 break; 286 case WRITE_CAPACITY_UNIT: 287 throttleBuilder.setWriteCapacityUnit(otherProto.getTimedQuota()); 288 break; 289 case ATOMIC_READ_SIZE: 290 throttleBuilder.setAtomicReadSize(otherProto.getTimedQuota()); 291 break; 292 case ATOMIC_REQUEST_NUMBER: 293 throttleBuilder.setAtomicReqNum(otherProto.getTimedQuota()); 294 break; 295 case ATOMIC_WRITE_SIZE: 296 throttleBuilder.setAtomicWriteSize(otherProto.getTimedQuota()); 297 break; 298 default: 299 } 300 } 301 } 302 303 // Propagate the space quota portion 304 QuotaProtos.SpaceQuota.Builder spaceBuilder = 305 (spaceProto == null ? null : spaceProto.toBuilder()); 306 if (other instanceof SpaceLimitSettings) { 307 if (spaceBuilder == null) { 308 spaceBuilder = QuotaProtos.SpaceQuota.newBuilder(); 309 } 310 SpaceLimitSettings settingsToMerge = (SpaceLimitSettings) other; 311 312 QuotaProtos.SpaceLimitRequest spaceRequest = settingsToMerge.getProto(); 313 314 // The message contained the expect SpaceQuota object 315 if (spaceRequest.hasQuota()) { 316 SpaceQuota quotaToMerge = spaceRequest.getQuota(); 317 // Validate that the two settings are for the same target. 318 // SpaceQuotas either apply to a table or a namespace (no user spacequota). 319 if ( 320 !Objects.equals(getTableName(), settingsToMerge.getTableName()) 321 && !Objects.equals(getNamespace(), settingsToMerge.getNamespace()) 322 ) { 323 throw new IllegalArgumentException("Cannot merge " + settingsToMerge + " into " + this); 324 } 325 326 if (quotaToMerge.getRemove()) { 327 // It means it's a remove request 328 // Update the builder to propagate the removal 329 spaceBuilder.setRemove(true).clearSoftLimit().clearViolationPolicy(); 330 } else { 331 // Add the new settings to the existing settings 332 spaceBuilder.mergeFrom(quotaToMerge); 333 } 334 } 335 } 336 337 boolean removeSpaceBuilder = 338 (spaceBuilder == null) || (spaceBuilder.hasRemove() && spaceBuilder.getRemove()); 339 340 Boolean bypassGlobals = this.bypassGlobals; 341 if (other instanceof QuotaGlobalsSettingsBypass) { 342 bypassGlobals = ((QuotaGlobalsSettingsBypass) other).getBypass(); 343 } 344 345 if (throttleBuilder == null && removeSpaceBuilder && bypassGlobals == null) { 346 return null; 347 } 348 349 return new GlobalQuotaSettingsImpl(getUserName(), getTableName(), getNamespace(), 350 getRegionServer(), (throttleBuilder == null ? null : throttleBuilder.build()), bypassGlobals, 351 (removeSpaceBuilder ? null : spaceBuilder.build())); 352 } 353 354 private void validateTimedQuota(final TimedQuota timedQuota) throws IOException { 355 if (timedQuota.getSoftLimit() < 1) { 356 throw new DoNotRetryIOException(new UnsupportedOperationException( 357 "The throttle limit must be greater then 0, got " + timedQuota.getSoftLimit())); 358 } 359 } 360 361 @Override 362 public String toString() { 363 StringBuilder builder = new StringBuilder(); 364 builder.append("GlobalQuota: "); 365 if (throttleProto != null) { 366 Map<ThrottleType, TimedQuota> throttleQuotas = buildThrottleQuotas(throttleProto); 367 builder.append(" { TYPE => THROTTLE "); 368 for (Entry<ThrottleType, TimedQuota> entry : throttleQuotas.entrySet()) { 369 final ThrottleType type = entry.getKey(); 370 final TimedQuota timedQuota = entry.getValue(); 371 builder.append("{THROTTLE_TYPE => ").append(type.name()).append(", LIMIT => "); 372 if (timedQuota.hasSoftLimit()) { 373 switch (type) { 374 case REQUEST_NUMBER: 375 case WRITE_NUMBER: 376 case READ_NUMBER: 377 case ATOMIC_REQUEST_NUMBER: 378 builder.append(String.format("%dreq", timedQuota.getSoftLimit())); 379 break; 380 case REQUEST_SIZE: 381 case WRITE_SIZE: 382 case READ_SIZE: 383 case ATOMIC_READ_SIZE: 384 case ATOMIC_WRITE_SIZE: 385 builder.append(sizeToString(timedQuota.getSoftLimit())); 386 break; 387 case REQUEST_CAPACITY_UNIT: 388 case READ_CAPACITY_UNIT: 389 case WRITE_CAPACITY_UNIT: 390 builder.append(String.format("%dCU", timedQuota.getSoftLimit())); 391 default: 392 } 393 } else if (timedQuota.hasShare()) { 394 builder.append(String.format("%.2f%%", timedQuota.getShare())); 395 } 396 builder.append('/'); 397 builder.append(timeToString(ProtobufUtil.toTimeUnit(timedQuota.getTimeUnit()))); 398 if (timedQuota.hasScope()) { 399 builder.append(", SCOPE => "); 400 builder.append(timedQuota.getScope().toString()); 401 } 402 } 403 builder.append("} } "); 404 } else { 405 builder.append(" {} "); 406 } 407 if (bypassGlobals != null) { 408 builder.append(" { GLOBAL_BYPASS => " + bypassGlobals + " } "); 409 } 410 if (spaceProto != null) { 411 builder.append(" { TYPE => SPACE"); 412 if (getTableName() != null) { 413 builder.append(", TABLE => ").append(getTableName()); 414 } 415 if (getNamespace() != null) { 416 builder.append(", NAMESPACE => ").append(getNamespace()); 417 } 418 if (spaceProto.getRemove()) { 419 builder.append(", REMOVE => ").append(spaceProto.getRemove()); 420 } else { 421 builder.append(", LIMIT => ").append(sizeToString(spaceProto.getSoftLimit())); 422 builder.append(", VIOLATION_POLICY => ").append(spaceProto.getViolationPolicy()); 423 } 424 builder.append(" } "); 425 } 426 return builder.toString(); 427 } 428 429 private Map<ThrottleType, TimedQuota> buildThrottleQuotas(Throttle proto) { 430 HashMap<ThrottleType, TimedQuota> quotas = new HashMap<>(); 431 if (proto.hasReadNum()) { 432 quotas.put(ThrottleType.READ_NUMBER, proto.getReadNum()); 433 } 434 if (proto.hasReadSize()) { 435 quotas.put(ThrottleType.READ_SIZE, proto.getReadSize()); 436 } 437 if (proto.hasReqNum()) { 438 quotas.put(ThrottleType.REQUEST_NUMBER, proto.getReqNum()); 439 } 440 if (proto.hasReqSize()) { 441 quotas.put(ThrottleType.REQUEST_SIZE, proto.getReqSize()); 442 } 443 if (proto.hasWriteNum()) { 444 quotas.put(ThrottleType.WRITE_NUMBER, proto.getWriteNum()); 445 } 446 if (proto.hasWriteSize()) { 447 quotas.put(ThrottleType.WRITE_SIZE, proto.getWriteSize()); 448 } 449 if (proto.hasReqCapacityUnit()) { 450 quotas.put(ThrottleType.REQUEST_CAPACITY_UNIT, proto.getReqCapacityUnit()); 451 } 452 if (proto.hasReadCapacityUnit()) { 453 quotas.put(ThrottleType.READ_CAPACITY_UNIT, proto.getReqCapacityUnit()); 454 } 455 if (proto.hasWriteCapacityUnit()) { 456 quotas.put(ThrottleType.WRITE_CAPACITY_UNIT, proto.getWriteCapacityUnit()); 457 } 458 return quotas; 459 } 460}